diff options
Diffstat (limited to 'clang/lib/CodeGen/TargetInfo.cpp')
| -rw-r--r-- | clang/lib/CodeGen/TargetInfo.cpp | 1734 |
1 files changed, 1379 insertions, 355 deletions
diff --git a/clang/lib/CodeGen/TargetInfo.cpp b/clang/lib/CodeGen/TargetInfo.cpp index 682ef18da73b..9cd63ebe29ee 100644 --- a/clang/lib/CodeGen/TargetInfo.cpp +++ b/clang/lib/CodeGen/TargetInfo.cpp @@ -20,6 +20,7 @@ #include "clang/AST/Attr.h" #include "clang/AST/RecordLayout.h" #include "clang/Basic/CodeGenOptions.h" +#include "clang/Basic/DiagnosticFrontend.h" #include "clang/CodeGen/CGFunctionInfo.h" #include "clang/CodeGen/SwiftCallingConv.h" #include "llvm/ADT/SmallBitVector.h" @@ -28,6 +29,7 @@ #include "llvm/ADT/Triple.h" #include "llvm/ADT/Twine.h" #include "llvm/IR/DataLayout.h" +#include "llvm/IR/IntrinsicsNVPTX.h" #include "llvm/IR/Type.h" #include "llvm/Support/raw_ostream.h" #include <algorithm> // std::sort @@ -96,6 +98,17 @@ Address ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, return Address::invalid(); } +bool ABIInfo::isPromotableIntegerTypeForABI(QualType Ty) const { + if (Ty->isPromotableIntegerType()) + return true; + + if (const auto *EIT = Ty->getAs<ExtIntType>()) + if (EIT->getNumBits() < getContext().getTypeSize(getContext().IntTy)) + return true; + + return false; +} + ABIInfo::~ABIInfo() {} /// Does the given lowering require more than the given number of @@ -384,7 +397,7 @@ static Address emitMergePHI(CodeGenFunction &CGF, return Address(PHI, Align); } -TargetCodeGenInfo::~TargetCodeGenInfo() { delete Info; } +TargetCodeGenInfo::~TargetCodeGenInfo() = default; // If someone can figure out a general rule for this, that would be great. // It's probably just doomed to be platform-dependent, though. @@ -486,11 +499,15 @@ static bool isEmptyField(ASTContext &Context, const FieldDecl *FD, // Constant arrays of empty records count as empty, strip them off. // Constant arrays of zero length always count as empty. + bool WasArray = false; if (AllowArrays) while (const ConstantArrayType *AT = Context.getAsConstantArrayType(FT)) { if (AT->getSize() == 0) return true; FT = AT->getElementType(); + // The [[no_unique_address]] special case below does not apply to + // arrays of C++ empty records, so we need to remember this fact. + WasArray = true; } const RecordType *RT = FT->getAs<RecordType>(); @@ -501,7 +518,14 @@ static bool isEmptyField(ASTContext &Context, const FieldDecl *FD, // // FIXME: We should use a predicate for whether this behavior is true in the // current ABI. - if (isa<CXXRecordDecl>(RT->getDecl())) + // + // The exception to the above rule are fields marked with the + // [[no_unique_address]] attribute (since C++20). Those do count as empty + // according to the Itanium ABI. The exception applies only to records, + // not arrays of records, so we must also check whether we stripped off an + // array type above. + if (isa<CXXRecordDecl>(RT->getDecl()) && + (WasArray || !FD->hasAttr<NoUniqueAddressAttr>())) return false; return isEmptyRecord(Context, FT, AllowArrays); @@ -681,7 +705,7 @@ public: class DefaultTargetCodeGenInfo : public TargetCodeGenInfo { public: DefaultTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT) - : TargetCodeGenInfo(new DefaultABIInfo(CGT)) {} + : TargetCodeGenInfo(std::make_unique<DefaultABIInfo>(CGT)) {} }; ABIArgInfo DefaultABIInfo::classifyArgumentType(QualType Ty) const { @@ -700,8 +724,16 @@ ABIArgInfo DefaultABIInfo::classifyArgumentType(QualType Ty) const { if (const EnumType *EnumTy = Ty->getAs<EnumType>()) Ty = EnumTy->getDecl()->getIntegerType(); - return (Ty->isPromotableIntegerType() ? ABIArgInfo::getExtend(Ty) - : ABIArgInfo::getDirect()); + ASTContext &Context = getContext(); + if (const auto *EIT = Ty->getAs<ExtIntType>()) + if (EIT->getNumBits() > + Context.getTypeSize(Context.getTargetInfo().hasInt128Type() + ? Context.Int128Ty + : Context.LongLongTy)) + return getNaturalAlignIndirect(Ty); + + return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty) + : ABIArgInfo::getDirect()); } ABIArgInfo DefaultABIInfo::classifyReturnType(QualType RetTy) const { @@ -715,8 +747,15 @@ ABIArgInfo DefaultABIInfo::classifyReturnType(QualType RetTy) const { if (const EnumType *EnumTy = RetTy->getAs<EnumType>()) RetTy = EnumTy->getDecl()->getIntegerType(); - return (RetTy->isPromotableIntegerType() ? ABIArgInfo::getExtend(RetTy) - : ABIArgInfo::getDirect()); + if (const auto *EIT = RetTy->getAs<ExtIntType>()) + if (EIT->getNumBits() > + getContext().getTypeSize(getContext().getTargetInfo().hasInt128Type() + ? getContext().Int128Ty + : getContext().LongLongTy)) + return getNaturalAlignIndirect(RetTy); + + return (isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy) + : ABIArgInfo::getDirect()); } //===----------------------------------------------------------------------===// @@ -726,11 +765,19 @@ ABIArgInfo DefaultABIInfo::classifyReturnType(QualType RetTy) const { //===----------------------------------------------------------------------===// class WebAssemblyABIInfo final : public SwiftABIInfo { +public: + enum ABIKind { + MVP = 0, + ExperimentalMV = 1, + }; + +private: DefaultABIInfo defaultInfo; + ABIKind Kind; public: - explicit WebAssemblyABIInfo(CodeGen::CodeGenTypes &CGT) - : SwiftABIInfo(CGT), defaultInfo(CGT) {} + explicit WebAssemblyABIInfo(CodeGen::CodeGenTypes &CGT, ABIKind Kind) + : SwiftABIInfo(CGT), defaultInfo(CGT), Kind(Kind) {} private: ABIArgInfo classifyReturnType(QualType RetTy) const; @@ -761,8 +808,9 @@ private: class WebAssemblyTargetCodeGenInfo final : public TargetCodeGenInfo { public: - explicit WebAssemblyTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT) - : TargetCodeGenInfo(new WebAssemblyABIInfo(CGT)) {} + explicit WebAssemblyTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, + WebAssemblyABIInfo::ABIKind K) + : TargetCodeGenInfo(std::make_unique<WebAssemblyABIInfo>(CGT, K)) {} void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const override { @@ -813,6 +861,20 @@ ABIArgInfo WebAssemblyABIInfo::classifyArgumentType(QualType Ty) const { // though watch out for things like bitfields. if (const Type *SeltTy = isSingleElementStruct(Ty, getContext())) return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0))); + // For the experimental multivalue ABI, fully expand all other aggregates + if (Kind == ABIKind::ExperimentalMV) { + const RecordType *RT = Ty->getAs<RecordType>(); + assert(RT); + bool HasBitField = false; + for (auto *Field : RT->getDecl()->fields()) { + if (Field->isBitField()) { + HasBitField = true; + break; + } + } + if (!HasBitField) + return ABIArgInfo::getExpand(); + } } // Otherwise just do the default thing. @@ -832,6 +894,9 @@ ABIArgInfo WebAssemblyABIInfo::classifyReturnType(QualType RetTy) const { // ABIArgInfo::getDirect(). if (const Type *SeltTy = isSingleElementStruct(RetTy, getContext())) return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0))); + // For the experimental multivalue ABI, return all other aggregates + if (Kind == ABIKind::ExperimentalMV) + return ABIArgInfo::getDirect(); } } @@ -871,8 +936,8 @@ class PNaClABIInfo : public ABIInfo { class PNaClTargetCodeGenInfo : public TargetCodeGenInfo { public: - PNaClTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT) - : TargetCodeGenInfo(new PNaClABIInfo(CGT)) {} + PNaClTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT) + : TargetCodeGenInfo(std::make_unique<PNaClABIInfo>(CGT)) {} }; void PNaClABIInfo::computeInfo(CGFunctionInfo &FI) const { @@ -906,10 +971,15 @@ ABIArgInfo PNaClABIInfo::classifyArgumentType(QualType Ty) const { } else if (Ty->isFloatingType()) { // Floating-point types don't go inreg. return ABIArgInfo::getDirect(); + } else if (const auto *EIT = Ty->getAs<ExtIntType>()) { + // Treat extended integers as integers if <=64, otherwise pass indirectly. + if (EIT->getNumBits() > 64) + return getNaturalAlignIndirect(Ty); + return ABIArgInfo::getDirect(); } - return (Ty->isPromotableIntegerType() ? ABIArgInfo::getExtend(Ty) - : ABIArgInfo::getDirect()); + return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty) + : ABIArgInfo::getDirect()); } ABIArgInfo PNaClABIInfo::classifyReturnType(QualType RetTy) const { @@ -920,12 +990,19 @@ ABIArgInfo PNaClABIInfo::classifyReturnType(QualType RetTy) const { if (isAggregateTypeForABI(RetTy)) return getNaturalAlignIndirect(RetTy); + // Treat extended integers as integers if <=64, otherwise pass indirectly. + if (const auto *EIT = RetTy->getAs<ExtIntType>()) { + if (EIT->getNumBits() > 64) + return getNaturalAlignIndirect(RetTy); + return ABIArgInfo::getDirect(); + } + // Treat an enum type as its underlying type. if (const EnumType *EnumTy = RetTy->getAs<EnumType>()) RetTy = EnumTy->getDecl()->getIntegerType(); - return (RetTy->isPromotableIntegerType() ? ABIArgInfo::getExtend(RetTy) - : ABIArgInfo::getDirect()); + return (isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy) + : ABIArgInfo::getDirect()); } /// IsX86_MMXType - Return true if this is an MMX type. @@ -943,7 +1020,8 @@ static llvm::Type* X86AdjustInlineAsmType(CodeGen::CodeGenFunction &CGF, .Cases("y", "&y", "^Ym", true) .Default(false); if (IsMMXCons && Ty->isVectorTy()) { - if (cast<llvm::VectorType>(Ty)->getBitWidth() != 64) { + if (cast<llvm::VectorType>(Ty)->getPrimitiveSizeInBits().getFixedSize() != + 64) { // Invalid MMX constraint return nullptr; } @@ -1112,7 +1190,7 @@ public: X86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI, bool RetSmallStructInRegABI, bool Win32StructABI, unsigned NumRegisterParameters, bool SoftFloatABI) - : TargetCodeGenInfo(new X86_32ABIInfo( + : TargetCodeGenInfo(std::make_unique<X86_32ABIInfo>( CGT, DarwinVectorABI, RetSmallStructInRegABI, Win32StructABI, NumRegisterParameters, SoftFloatABI)) {} @@ -1412,8 +1490,8 @@ ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy, // registers and we need to make sure to pick a type the LLVM // backend will like. if (Size == 128) - return ABIArgInfo::getDirect(llvm::VectorType::get( - llvm::Type::getInt64Ty(getVMContext()), 2)); + return ABIArgInfo::getDirect(llvm::FixedVectorType::get( + llvm::Type::getInt64Ty(getVMContext()), 2)); // Always return in register if it fits in a general purpose // register, or if it is 64 bits and has a single element. @@ -1470,15 +1548,19 @@ ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy, if (const EnumType *EnumTy = RetTy->getAs<EnumType>()) RetTy = EnumTy->getDecl()->getIntegerType(); - return (RetTy->isPromotableIntegerType() ? ABIArgInfo::getExtend(RetTy) - : ABIArgInfo::getDirect()); + if (const auto *EIT = RetTy->getAs<ExtIntType>()) + if (EIT->getNumBits() > 64) + return getIndirectReturnResult(RetTy, State); + + return (isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy) + : ABIArgInfo::getDirect()); } -static bool isSSEVectorType(ASTContext &Context, QualType Ty) { +static bool isSIMDVectorType(ASTContext &Context, QualType Ty) { return Ty->getAs<VectorType>() && Context.getTypeSize(Ty) == 128; } -static bool isRecordWithSSEVectorType(ASTContext &Context, QualType Ty) { +static bool isRecordWithSIMDVectorType(ASTContext &Context, QualType Ty) { const RecordType *RT = Ty->getAs<RecordType>(); if (!RT) return 0; @@ -1487,16 +1569,16 @@ static bool isRecordWithSSEVectorType(ASTContext &Context, QualType Ty) { // If this is a C++ record, check the bases first. if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) for (const auto &I : CXXRD->bases()) - if (!isRecordWithSSEVectorType(Context, I.getType())) + if (!isRecordWithSIMDVectorType(Context, I.getType())) return false; for (const auto *i : RD->fields()) { QualType FT = i->getType(); - if (isSSEVectorType(Context, FT)) + if (isSIMDVectorType(Context, FT)) return true; - if (isRecordWithSSEVectorType(Context, FT)) + if (isRecordWithSIMDVectorType(Context, FT)) return true; } @@ -1517,8 +1599,8 @@ unsigned X86_32ABIInfo::getTypeStackAlignInBytes(QualType Ty, } // Otherwise, if the type contains an SSE vector type, the alignment is 16. - if (Align >= 16 && (isSSEVectorType(getContext(), Ty) || - isRecordWithSSEVectorType(getContext(), Ty))) + if (Align >= 16 && (isSIMDVectorType(getContext(), Ty) || + isRecordWithSIMDVectorType(getContext(), Ty))) return 16; return MinABIStackAlignInBytes; @@ -1661,7 +1743,7 @@ void X86_32ABIInfo::runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) c isHomogeneousAggregate(Ty, Base, NumElts)) { if (State.FreeSSERegs >= NumElts) { State.FreeSSERegs -= NumElts; - Args[I].info = ABIArgInfo::getDirect(); + Args[I].info = ABIArgInfo::getDirectInReg(); State.IsPreassigned.set(I); } } @@ -1676,6 +1758,7 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty, bool IsVectorCall = State.CC == llvm::CallingConv::X86_VectorCall; Ty = useFirstFieldIfTransparentUnion(Ty); + TypeInfo TI = getContext().getTypeInfo(Ty); // Check with the C++ ABI first. const RecordType *RT = Ty->getAs<RecordType>(); @@ -1725,7 +1808,7 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty, bool NeedsPadding = false; bool InReg; if (shouldAggregateUseDirect(Ty, State, InReg, NeedsPadding)) { - unsigned SizeInRegs = (getContext().getTypeSize(Ty) + 31) / 32; + unsigned SizeInRegs = (TI.Width + 31) / 32; SmallVector<llvm::Type*, 3> Elements(SizeInRegs, Int32); llvm::Type *Result = llvm::StructType::get(LLVMContext, Elements); if (InReg) @@ -1735,14 +1818,19 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty, } llvm::IntegerType *PaddingType = NeedsPadding ? Int32 : nullptr; + // Pass over-aligned aggregates on Windows indirectly. This behavior was + // added in MSVC 2015. + if (IsWin32StructABI && TI.AlignIsRequired && TI.Align > 32) + return getIndirectResult(Ty, /*ByVal=*/false, State); + // Expand small (<= 128-bit) record types when we know that the stack layout // of those arguments will match the struct. This is important because the // LLVM backend isn't smart enough to remove byval, which inhibits many // optimizations. // Don't do this for the MCU if there are still free integer registers // (see X86_64 ABI for full explanation). - if (getContext().getTypeSize(Ty) <= 4 * 32 && - (!IsMCUABI || State.FreeRegs == 0) && canExpandIndirectArgument(Ty)) + if (TI.Width <= 4 * 32 && (!IsMCUABI || State.FreeRegs == 0) && + canExpandIndirectArgument(Ty)) return ABIArgInfo::getExpandWithPadding( IsFastCall || IsVectorCall || IsRegCall, PaddingType); @@ -1750,14 +1838,24 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty, } if (const VectorType *VT = Ty->getAs<VectorType>()) { + // On Windows, vectors are passed directly if registers are available, or + // indirectly if not. This avoids the need to align argument memory. Pass + // user-defined vector types larger than 512 bits indirectly for simplicity. + if (IsWin32StructABI) { + if (TI.Width <= 512 && State.FreeSSERegs > 0) { + --State.FreeSSERegs; + return ABIArgInfo::getDirectInReg(); + } + return getIndirectResult(Ty, /*ByVal=*/false, State); + } + // On Darwin, some vectors are passed in memory, we handle this by passing // it as an i8/i16/i32/i64. if (IsDarwinVectorABI) { - uint64_t Size = getContext().getTypeSize(Ty); - if ((Size == 8 || Size == 16 || Size == 32) || - (Size == 64 && VT->getNumElements() == 1)) - return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), - Size)); + if ((TI.Width == 8 || TI.Width == 16 || TI.Width == 32) || + (TI.Width == 64 && VT->getNumElements() == 1)) + return ABIArgInfo::getDirect( + llvm::IntegerType::get(getVMContext(), TI.Width)); } if (IsX86_MMXType(CGT.ConvertType(Ty))) @@ -1772,12 +1870,21 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty, bool InReg = shouldPrimitiveUseInReg(Ty, State); - if (Ty->isPromotableIntegerType()) { + if (isPromotableIntegerTypeForABI(Ty)) { if (InReg) return ABIArgInfo::getExtendInReg(Ty); return ABIArgInfo::getExtend(Ty); } + if (const auto * EIT = Ty->getAs<ExtIntType>()) { + if (EIT->getNumBits() <= 64) { + if (InReg) + return ABIArgInfo::getDirectInReg(); + return ABIArgInfo::getDirect(); + } + return getIndirectResult(Ty, /*ByVal=*/false, State); + } + if (InReg) return ABIArgInfo::getDirectInReg(); return ABIArgInfo::getDirect(); @@ -1787,9 +1894,10 @@ void X86_32ABIInfo::computeInfo(CGFunctionInfo &FI) const { CCState State(FI); if (IsMCUABI) State.FreeRegs = 3; - else if (State.CC == llvm::CallingConv::X86_FastCall) + else if (State.CC == llvm::CallingConv::X86_FastCall) { State.FreeRegs = 2; - else if (State.CC == llvm::CallingConv::X86_VectorCall) { + State.FreeSSERegs = 3; + } else if (State.CC == llvm::CallingConv::X86_VectorCall) { State.FreeRegs = 2; State.FreeSSERegs = 6; } else if (FI.getHasRegParm()) @@ -1797,6 +1905,11 @@ void X86_32ABIInfo::computeInfo(CGFunctionInfo &FI) const { else if (State.CC == llvm::CallingConv::X86_RegCall) { State.FreeRegs = 5; State.FreeSSERegs = 8; + } else if (IsWin32StructABI) { + // Since MSVC 2015, the first three SSE vectors have been passed in + // registers. The rest are passed indirectly. + State.FreeRegs = DefaultNumRegisterParameters; + State.FreeSSERegs = 3; } else State.FreeRegs = DefaultNumRegisterParameters; @@ -1843,16 +1956,25 @@ X86_32ABIInfo::addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields, CharUnits &StackOffset, ABIArgInfo &Info, QualType Type) const { // Arguments are always 4-byte-aligned. - CharUnits FieldAlign = CharUnits::fromQuantity(4); + CharUnits WordSize = CharUnits::fromQuantity(4); + assert(StackOffset.isMultipleOf(WordSize) && "unaligned inalloca struct"); - assert(StackOffset.isMultipleOf(FieldAlign) && "unaligned inalloca struct"); - Info = ABIArgInfo::getInAlloca(FrameFields.size()); - FrameFields.push_back(CGT.ConvertTypeForMem(Type)); - StackOffset += getContext().getTypeSizeInChars(Type); + // sret pointers and indirect things will require an extra pointer + // indirection, unless they are byval. Most things are byval, and will not + // require this indirection. + bool IsIndirect = false; + if (Info.isIndirect() && !Info.getIndirectByVal()) + IsIndirect = true; + Info = ABIArgInfo::getInAlloca(FrameFields.size(), IsIndirect); + llvm::Type *LLTy = CGT.ConvertTypeForMem(Type); + if (IsIndirect) + LLTy = LLTy->getPointerTo(0); + FrameFields.push_back(LLTy); + StackOffset += IsIndirect ? WordSize : getContext().getTypeSizeInChars(Type); // Insert padding bytes to respect alignment. CharUnits FieldEnd = StackOffset; - StackOffset = FieldEnd.alignTo(FieldAlign); + StackOffset = FieldEnd.alignTo(WordSize); if (StackOffset != FieldEnd) { CharUnits NumBytes = StackOffset - FieldEnd; llvm::Type *Ty = llvm::Type::getInt8Ty(getVMContext()); @@ -1866,16 +1988,12 @@ static bool isArgInAlloca(const ABIArgInfo &Info) { switch (Info.getKind()) { case ABIArgInfo::InAlloca: return true; - case ABIArgInfo::Indirect: - assert(Info.getIndirectByVal()); - return true; case ABIArgInfo::Ignore: return false; + case ABIArgInfo::Indirect: case ABIArgInfo::Direct: case ABIArgInfo::Extend: - if (Info.getInReg()) - return false; - return true; + return !Info.getInReg(); case ABIArgInfo::Expand: case ABIArgInfo::CoerceAndExpand: // These are aggregate types which are never passed in registers when @@ -1909,8 +2027,7 @@ void X86_32ABIInfo::rewriteWithInAlloca(CGFunctionInfo &FI) const { // Put the sret parameter into the inalloca struct if it's in memory. if (Ret.isIndirect() && !Ret.getInReg()) { - CanQualType PtrTy = getContext().getPointerType(FI.getReturnType()); - addFieldToArgStruct(FrameFields, StackOffset, Ret, PtrTy); + addFieldToArgStruct(FrameFields, StackOffset, Ret, FI.getReturnType()); // On Windows, the hidden sret parameter is always returned in eax. Ret.setInAllocaSRet(IsWin32StructABI); } @@ -2207,7 +2324,7 @@ public: if (info.isDirect()) { llvm::Type *ty = info.getCoerceToType(); if (llvm::VectorType *vectorTy = dyn_cast_or_null<llvm::VectorType>(ty)) - return (vectorTy->getBitWidth() > 128); + return vectorTy->getPrimitiveSizeInBits().getFixedSize() > 128; } return false; } @@ -2280,7 +2397,7 @@ private: class X86_64TargetCodeGenInfo : public TargetCodeGenInfo { public: X86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel) - : TargetCodeGenInfo(new X86_64ABIInfo(CGT, AVXLevel)) {} + : TargetCodeGenInfo(std::make_unique<X86_64ABIInfo>(CGT, AVXLevel)) {} const X86_64ABIInfo &getABIInfo() const { return static_cast<const X86_64ABIInfo&>(TargetCodeGenInfo::getABIInfo()); @@ -2361,8 +2478,110 @@ public: } } } + + void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc, + const FunctionDecl *Caller, + const FunctionDecl *Callee, + const CallArgList &Args) const override; }; +static void initFeatureMaps(const ASTContext &Ctx, + llvm::StringMap<bool> &CallerMap, + const FunctionDecl *Caller, + llvm::StringMap<bool> &CalleeMap, + const FunctionDecl *Callee) { + if (CalleeMap.empty() && CallerMap.empty()) { + // The caller is potentially nullptr in the case where the call isn't in a + // function. In this case, the getFunctionFeatureMap ensures we just get + // the TU level setting (since it cannot be modified by 'target'.. + Ctx.getFunctionFeatureMap(CallerMap, Caller); + Ctx.getFunctionFeatureMap(CalleeMap, Callee); + } +} + +static bool checkAVXParamFeature(DiagnosticsEngine &Diag, + SourceLocation CallLoc, + const llvm::StringMap<bool> &CallerMap, + const llvm::StringMap<bool> &CalleeMap, + QualType Ty, StringRef Feature, + bool IsArgument) { + bool CallerHasFeat = CallerMap.lookup(Feature); + bool CalleeHasFeat = CalleeMap.lookup(Feature); + if (!CallerHasFeat && !CalleeHasFeat) + return Diag.Report(CallLoc, diag::warn_avx_calling_convention) + << IsArgument << Ty << Feature; + + // Mixing calling conventions here is very clearly an error. + if (!CallerHasFeat || !CalleeHasFeat) + return Diag.Report(CallLoc, diag::err_avx_calling_convention) + << IsArgument << Ty << Feature; + + // Else, both caller and callee have the required feature, so there is no need + // to diagnose. + return false; +} + +static bool checkAVXParam(DiagnosticsEngine &Diag, ASTContext &Ctx, + SourceLocation CallLoc, + const llvm::StringMap<bool> &CallerMap, + const llvm::StringMap<bool> &CalleeMap, QualType Ty, + bool IsArgument) { + uint64_t Size = Ctx.getTypeSize(Ty); + if (Size > 256) + return checkAVXParamFeature(Diag, CallLoc, CallerMap, CalleeMap, Ty, + "avx512f", IsArgument); + + if (Size > 128) + return checkAVXParamFeature(Diag, CallLoc, CallerMap, CalleeMap, Ty, "avx", + IsArgument); + + return false; +} + +void X86_64TargetCodeGenInfo::checkFunctionCallABI( + CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller, + const FunctionDecl *Callee, const CallArgList &Args) const { + llvm::StringMap<bool> CallerMap; + llvm::StringMap<bool> CalleeMap; + unsigned ArgIndex = 0; + + // We need to loop through the actual call arguments rather than the the + // function's parameters, in case this variadic. + for (const CallArg &Arg : Args) { + // The "avx" feature changes how vectors >128 in size are passed. "avx512f" + // additionally changes how vectors >256 in size are passed. Like GCC, we + // warn when a function is called with an argument where this will change. + // Unlike GCC, we also error when it is an obvious ABI mismatch, that is, + // the caller and callee features are mismatched. + // Unfortunately, we cannot do this diagnostic in SEMA, since the callee can + // change its ABI with attribute-target after this call. + if (Arg.getType()->isVectorType() && + CGM.getContext().getTypeSize(Arg.getType()) > 128) { + initFeatureMaps(CGM.getContext(), CallerMap, Caller, CalleeMap, Callee); + QualType Ty = Arg.getType(); + // The CallArg seems to have desugared the type already, so for clearer + // diagnostics, replace it with the type in the FunctionDecl if possible. + if (ArgIndex < Callee->getNumParams()) + Ty = Callee->getParamDecl(ArgIndex)->getType(); + + if (checkAVXParam(CGM.getDiags(), CGM.getContext(), CallLoc, CallerMap, + CalleeMap, Ty, /*IsArgument*/ true)) + return; + } + ++ArgIndex; + } + + // Check return always, as we don't have a good way of knowing in codegen + // whether this value is used, tail-called, etc. + if (Callee->getReturnType()->isVectorType() && + CGM.getContext().getTypeSize(Callee->getReturnType()) > 128) { + initFeatureMaps(CGM.getContext(), CallerMap, Caller, CalleeMap, Callee); + checkAVXParam(CGM.getDiags(), CGM.getContext(), CallLoc, CallerMap, + CalleeMap, Callee->getReturnType(), + /*IsArgument*/ false); + } +} + static std::string qualifyWindowsLibrary(llvm::StringRef Lib) { // If the argument does not end in .lib, automatically add the suffix. // If the argument contains a space, enclose it in quotes. @@ -2424,7 +2643,7 @@ class WinX86_64TargetCodeGenInfo : public TargetCodeGenInfo { public: WinX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel) - : TargetCodeGenInfo(new WinX86_64ABIInfo(CGT, AVXLevel)) {} + : TargetCodeGenInfo(std::make_unique<WinX86_64ABIInfo>(CGT, AVXLevel)) {} void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const override; @@ -2731,6 +2950,15 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, return; } + if (const auto *EITy = Ty->getAs<ExtIntType>()) { + if (EITy->getNumBits() <= 64) + Current = Integer; + else if (EITy->getNumBits() <= 128) + Lo = Hi = Integer; + // Larger values need to get passed in memory. + return; + } + if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) { // Arrays are treated like structures. @@ -2905,8 +3133,11 @@ ABIArgInfo X86_64ABIInfo::getIndirectReturnResult(QualType Ty) const { if (const EnumType *EnumTy = Ty->getAs<EnumType>()) Ty = EnumTy->getDecl()->getIntegerType(); - return (Ty->isPromotableIntegerType() ? ABIArgInfo::getExtend(Ty) - : ABIArgInfo::getDirect()); + if (Ty->isExtIntType()) + return getNaturalAlignIndirect(Ty); + + return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty) + : ABIArgInfo::getDirect()); } return getNaturalAlignIndirect(Ty); @@ -2938,13 +3169,14 @@ ABIArgInfo X86_64ABIInfo::getIndirectResult(QualType Ty, // the argument in the free register. This does not seem to happen currently, // but this code would be much safer if we could mark the argument with // 'onstack'. See PR12193. - if (!isAggregateTypeForABI(Ty) && !IsIllegalVectorType(Ty)) { + if (!isAggregateTypeForABI(Ty) && !IsIllegalVectorType(Ty) && + !Ty->isExtIntType()) { // Treat an enum type as its underlying type. if (const EnumType *EnumTy = Ty->getAs<EnumType>()) Ty = EnumTy->getDecl()->getIntegerType(); - return (Ty->isPromotableIntegerType() ? ABIArgInfo::getExtend(Ty) - : ABIArgInfo::getDirect()); + return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty) + : ABIArgInfo::getDirect()); } if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) @@ -3001,11 +3233,11 @@ llvm::Type *X86_64ABIInfo::GetByteVectorType(QualType Ty) const { // Don't pass vXi128 vectors in their native type, the backend can't // legalize them. if (passInt128VectorsInMem() && - IRType->getVectorElementType()->isIntegerTy(128)) { + cast<llvm::VectorType>(IRType)->getElementType()->isIntegerTy(128)) { // Use a vXi64 vector. uint64_t Size = getContext().getTypeSize(Ty); - return llvm::VectorType::get(llvm::Type::getInt64Ty(getVMContext()), - Size / 64); + return llvm::FixedVectorType::get(llvm::Type::getInt64Ty(getVMContext()), + Size / 64); } return IRType; @@ -3020,8 +3252,8 @@ llvm::Type *X86_64ABIInfo::GetByteVectorType(QualType Ty) const { // Return a LLVM IR vector type based on the size of 'Ty'. - return llvm::VectorType::get(llvm::Type::getDoubleTy(getVMContext()), - Size / 64); + return llvm::FixedVectorType::get(llvm::Type::getDoubleTy(getVMContext()), + Size / 64); } /// BitsContainNoUserData - Return true if the specified [start,end) bit range @@ -3155,7 +3387,8 @@ GetSSETypeAtOffset(llvm::Type *IRType, unsigned IROffset, // case. if (ContainsFloatAtOffset(IRType, IROffset, getDataLayout()) && ContainsFloatAtOffset(IRType, IROffset+4, getDataLayout())) - return llvm::VectorType::get(llvm::Type::getFloatTy(getVMContext()), 2); + return llvm::FixedVectorType::get(llvm::Type::getFloatTy(getVMContext()), + 2); return llvm::Type::getDoubleTy(getVMContext()); } @@ -3326,7 +3559,7 @@ classifyReturnType(QualType RetTy) const { RetTy = EnumTy->getDecl()->getIntegerType(); if (RetTy->isIntegralOrEnumerationType() && - RetTy->isPromotableIntegerType()) + isPromotableIntegerTypeForABI(RetTy)) return ABIArgInfo::getExtend(RetTy); } break; @@ -3471,7 +3704,7 @@ ABIArgInfo X86_64ABIInfo::classifyArgumentType( Ty = EnumTy->getDecl()->getIntegerType(); if (Ty->isIntegralOrEnumerationType() && - Ty->isPromotableIntegerType()) + isPromotableIntegerTypeForABI(Ty)) return ABIArgInfo::getExtend(Ty); } @@ -3627,14 +3860,15 @@ void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const { } else { FI.getReturnInfo() = getIndirectReturnResult(FI.getReturnType()); } - } else if (IsRegCall && FI.getReturnType()->getAs<ComplexType>()) { + } else if (IsRegCall && FI.getReturnType()->getAs<ComplexType>() && + getContext().getCanonicalType(FI.getReturnType() + ->getAs<ComplexType>() + ->getElementType()) == + getContext().LongDoubleTy) // Complex Long Double Type is passed in Memory when Regcall // calling convention is used. - const ComplexType *CT = FI.getReturnType()->getAs<ComplexType>(); - if (getContext().getCanonicalType(CT->getElementType()) == - getContext().LongDoubleTy) - FI.getReturnInfo() = getIndirectReturnResult(FI.getReturnType()); - } else + FI.getReturnInfo() = getIndirectReturnResult(FI.getReturnType()); + else FI.getReturnInfo() = classifyReturnType(FI.getReturnType()); } @@ -4021,14 +4255,25 @@ ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs, // Mingw64 GCC returns i128 in XMM0. Coerce to v2i64 to handle that. // Clang matches them for compatibility. - return ABIArgInfo::getDirect( - llvm::VectorType::get(llvm::Type::getInt64Ty(getVMContext()), 2)); + return ABIArgInfo::getDirect(llvm::FixedVectorType::get( + llvm::Type::getInt64Ty(getVMContext()), 2)); default: break; } } + if (Ty->isExtIntType()) { + // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is + // not 1, 2, 4, or 8 bytes, must be passed by reference." + // However, non-power-of-two _ExtInts will be passed as 1,2,4 or 8 bytes + // anyway as long is it fits in them, so we don't have to check the power of + // 2. + if (Width <= 64) + return ABIArgInfo::getDirect(); + return ABIArgInfo::getIndirect(Align, /*ByVal=*/false); + } + return ABIArgInfo::getDirect(); } @@ -4118,17 +4363,247 @@ Address WinX86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, /*allowHigherAlign*/ false); } +static bool PPC_initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF, + llvm::Value *Address, bool Is64Bit, + bool IsAIX) { + // This is calculated from the LLVM and GCC tables and verified + // against gcc output. AFAIK all PPC ABIs use the same encoding. + + CodeGen::CGBuilderTy &Builder = CGF.Builder; + + llvm::IntegerType *i8 = CGF.Int8Ty; + llvm::Value *Four8 = llvm::ConstantInt::get(i8, 4); + llvm::Value *Eight8 = llvm::ConstantInt::get(i8, 8); + llvm::Value *Sixteen8 = llvm::ConstantInt::get(i8, 16); + + // 0-31: r0-31, the 4-byte or 8-byte general-purpose registers + AssignToArrayRange(Builder, Address, Is64Bit ? Eight8 : Four8, 0, 31); + + // 32-63: fp0-31, the 8-byte floating-point registers + AssignToArrayRange(Builder, Address, Eight8, 32, 63); + + // 64-67 are various 4-byte or 8-byte special-purpose registers: + // 64: mq + // 65: lr + // 66: ctr + // 67: ap + AssignToArrayRange(Builder, Address, Is64Bit ? Eight8 : Four8, 64, 67); + + // 68-76 are various 4-byte special-purpose registers: + // 68-75 cr0-7 + // 76: xer + AssignToArrayRange(Builder, Address, Four8, 68, 76); + + // 77-108: v0-31, the 16-byte vector registers + AssignToArrayRange(Builder, Address, Sixteen8, 77, 108); + + // 109: vrsave + // 110: vscr + AssignToArrayRange(Builder, Address, Is64Bit ? Eight8 : Four8, 109, 110); + + // AIX does not utilize the rest of the registers. + if (IsAIX) + return false; + + // 111: spe_acc + // 112: spefscr + // 113: sfp + AssignToArrayRange(Builder, Address, Is64Bit ? Eight8 : Four8, 111, 113); + + if (!Is64Bit) + return false; + + // TODO: Need to verify if these registers are used on 64 bit AIX with Power8 + // or above CPU. + // 64-bit only registers: + // 114: tfhar + // 115: tfiar + // 116: texasr + AssignToArrayRange(Builder, Address, Eight8, 114, 116); + + return false; +} + +// AIX +namespace { +/// AIXABIInfo - The AIX XCOFF ABI information. +class AIXABIInfo : public ABIInfo { + const bool Is64Bit; + const unsigned PtrByteSize; + CharUnits getParamTypeAlignment(QualType Ty) const; + +public: + AIXABIInfo(CodeGen::CodeGenTypes &CGT, bool Is64Bit) + : ABIInfo(CGT), Is64Bit(Is64Bit), PtrByteSize(Is64Bit ? 8 : 4) {} + + bool isPromotableTypeForABI(QualType Ty) const; + + ABIArgInfo classifyReturnType(QualType RetTy) const; + ABIArgInfo classifyArgumentType(QualType Ty) const; + + void computeInfo(CGFunctionInfo &FI) const override { + if (!getCXXABI().classifyReturnType(FI)) + FI.getReturnInfo() = classifyReturnType(FI.getReturnType()); + + for (auto &I : FI.arguments()) + I.info = classifyArgumentType(I.type); + } + + Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const override; +}; + +class AIXTargetCodeGenInfo : public TargetCodeGenInfo { + const bool Is64Bit; + +public: + AIXTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool Is64Bit) + : TargetCodeGenInfo(std::make_unique<AIXABIInfo>(CGT, Is64Bit)), + Is64Bit(Is64Bit) {} + int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override { + return 1; // r1 is the dedicated stack pointer + } + + bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF, + llvm::Value *Address) const override; +}; +} // namespace + +// Return true if the ABI requires Ty to be passed sign- or zero- +// extended to 32/64 bits. +bool AIXABIInfo::isPromotableTypeForABI(QualType Ty) const { + // Treat an enum type as its underlying type. + if (const EnumType *EnumTy = Ty->getAs<EnumType>()) + Ty = EnumTy->getDecl()->getIntegerType(); + + // Promotable integer types are required to be promoted by the ABI. + if (Ty->isPromotableIntegerType()) + return true; + + if (!Is64Bit) + return false; + + // For 64 bit mode, in addition to the usual promotable integer types, we also + // need to extend all 32-bit types, since the ABI requires promotion to 64 + // bits. + if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) + switch (BT->getKind()) { + case BuiltinType::Int: + case BuiltinType::UInt: + return true; + default: + break; + } + + return false; +} + +ABIArgInfo AIXABIInfo::classifyReturnType(QualType RetTy) const { + if (RetTy->isAnyComplexType()) + llvm::report_fatal_error("complex type is not supported on AIX yet"); + + if (RetTy->isVectorType()) + llvm::report_fatal_error("vector type is not supported on AIX yet"); + + if (RetTy->isVoidType()) + return ABIArgInfo::getIgnore(); + + // TODO: Evaluate if AIX power alignment rule would have an impact on the + // alignment here. + if (isAggregateTypeForABI(RetTy)) + return getNaturalAlignIndirect(RetTy); + + return (isPromotableTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy) + : ABIArgInfo::getDirect()); +} + +ABIArgInfo AIXABIInfo::classifyArgumentType(QualType Ty) const { + Ty = useFirstFieldIfTransparentUnion(Ty); + + if (Ty->isAnyComplexType()) + llvm::report_fatal_error("complex type is not supported on AIX yet"); + + if (Ty->isVectorType()) + llvm::report_fatal_error("vector type is not supported on AIX yet"); + + // TODO: Evaluate if AIX power alignment rule would have an impact on the + // alignment here. + if (isAggregateTypeForABI(Ty)) { + // Records with non-trivial destructors/copy-constructors should not be + // passed by value. + if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) + return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory); + + CharUnits CCAlign = getParamTypeAlignment(Ty); + CharUnits TyAlign = getContext().getTypeAlignInChars(Ty); + + return ABIArgInfo::getIndirect(CCAlign, /*ByVal*/ true, + /*Realign*/ TyAlign > CCAlign); + } + + return (isPromotableTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty) + : ABIArgInfo::getDirect()); +} + +CharUnits AIXABIInfo::getParamTypeAlignment(QualType Ty) const { + if (Ty->isAnyComplexType()) + llvm::report_fatal_error("complex type is not supported on AIX yet"); + + if (Ty->isVectorType()) + llvm::report_fatal_error("vector type is not supported on AIX yet"); + + // If the structure contains a vector type, the alignment is 16. + if (isRecordWithSIMDVectorType(getContext(), Ty)) + return CharUnits::fromQuantity(16); + + return CharUnits::fromQuantity(PtrByteSize); +} + +Address AIXABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const { + if (Ty->isAnyComplexType()) + llvm::report_fatal_error("complex type is not supported on AIX yet"); + + if (Ty->isVectorType()) + llvm::report_fatal_error("vector type is not supported on AIX yet"); + + auto TypeInfo = getContext().getTypeInfoInChars(Ty); + TypeInfo.second = getParamTypeAlignment(Ty); + + CharUnits SlotSize = CharUnits::fromQuantity(PtrByteSize); + + return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false, TypeInfo, + SlotSize, /*AllowHigher*/ true); +} + +bool AIXTargetCodeGenInfo::initDwarfEHRegSizeTable( + CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const { + return PPC_initDwarfEHRegSizeTable(CGF, Address, Is64Bit, /*IsAIX*/ true); +} + // PowerPC-32 namespace { /// PPC32_SVR4_ABIInfo - The 32-bit PowerPC ELF (SVR4) ABI information. class PPC32_SVR4_ABIInfo : public DefaultABIInfo { bool IsSoftFloatABI; + bool IsRetSmallStructInRegABI; CharUnits getParamTypeAlignment(QualType Ty) const; public: - PPC32_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT, bool SoftFloatABI) - : DefaultABIInfo(CGT), IsSoftFloatABI(SoftFloatABI) {} + PPC32_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT, bool SoftFloatABI, + bool RetSmallStructInRegABI) + : DefaultABIInfo(CGT), IsSoftFloatABI(SoftFloatABI), + IsRetSmallStructInRegABI(RetSmallStructInRegABI) {} + + ABIArgInfo classifyReturnType(QualType RetTy) const; + + void computeInfo(CGFunctionInfo &FI) const override { + if (!getCXXABI().classifyReturnType(FI)) + FI.getReturnInfo() = classifyReturnType(FI.getReturnType()); + for (auto &I : FI.arguments()) + I.info = classifyArgumentType(I.type); + } Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty) const override; @@ -4136,8 +4611,13 @@ public: class PPC32TargetCodeGenInfo : public TargetCodeGenInfo { public: - PPC32TargetCodeGenInfo(CodeGenTypes &CGT, bool SoftFloatABI) - : TargetCodeGenInfo(new PPC32_SVR4_ABIInfo(CGT, SoftFloatABI)) {} + PPC32TargetCodeGenInfo(CodeGenTypes &CGT, bool SoftFloatABI, + bool RetSmallStructInRegABI) + : TargetCodeGenInfo(std::make_unique<PPC32_SVR4_ABIInfo>( + CGT, SoftFloatABI, RetSmallStructInRegABI)) {} + + static bool isStructReturnInRegABI(const llvm::Triple &Triple, + const CodeGenOptions &Opts); int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override { // This is recovered from gcc output. @@ -4150,7 +4630,7 @@ public: } CharUnits PPC32_SVR4_ABIInfo::getParamTypeAlignment(QualType Ty) const { - // Complex types are passed just like their elements + // Complex types are passed just like their elements. if (const ComplexType *CTy = Ty->getAs<ComplexType>()) Ty = CTy->getElementType(); @@ -4173,6 +4653,34 @@ CharUnits PPC32_SVR4_ABIInfo::getParamTypeAlignment(QualType Ty) const { return CharUnits::fromQuantity(4); } +ABIArgInfo PPC32_SVR4_ABIInfo::classifyReturnType(QualType RetTy) const { + uint64_t Size; + + // -msvr4-struct-return puts small aggregates in GPR3 and GPR4. + if (isAggregateTypeForABI(RetTy) && IsRetSmallStructInRegABI && + (Size = getContext().getTypeSize(RetTy)) <= 64) { + // System V ABI (1995), page 3-22, specified: + // > A structure or union whose size is less than or equal to 8 bytes + // > shall be returned in r3 and r4, as if it were first stored in the + // > 8-byte aligned memory area and then the low addressed word were + // > loaded into r3 and the high-addressed word into r4. Bits beyond + // > the last member of the structure or union are not defined. + // + // GCC for big-endian PPC32 inserts the pad before the first member, + // not "beyond the last member" of the struct. To stay compatible + // with GCC, we coerce the struct to an integer of the same size. + // LLVM will extend it and return i32 in r3, or i64 in r3:r4. + if (Size == 0) + return ABIArgInfo::getIgnore(); + else { + llvm::Type *CoerceTy = llvm::Type::getIntNTy(getVMContext(), Size); + return ABIArgInfo::getDirect(CoerceTy); + } + } + + return DefaultABIInfo::classifyReturnType(RetTy); +} + // TODO: this implementation is now likely redundant with // DefaultABIInfo::EmitVAArg. Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList, @@ -4328,47 +4836,32 @@ Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList, return Result; } -bool -PPC32TargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF, - llvm::Value *Address) const { - // This is calculated from the LLVM and GCC tables and verified - // against gcc output. AFAIK all ABIs use the same encoding. - - CodeGen::CGBuilderTy &Builder = CGF.Builder; - - llvm::IntegerType *i8 = CGF.Int8Ty; - llvm::Value *Four8 = llvm::ConstantInt::get(i8, 4); - llvm::Value *Eight8 = llvm::ConstantInt::get(i8, 8); - llvm::Value *Sixteen8 = llvm::ConstantInt::get(i8, 16); - - // 0-31: r0-31, the 4-byte general-purpose registers - AssignToArrayRange(Builder, Address, Four8, 0, 31); - - // 32-63: fp0-31, the 8-byte floating-point registers - AssignToArrayRange(Builder, Address, Eight8, 32, 63); - - // 64-76 are various 4-byte special-purpose registers: - // 64: mq - // 65: lr - // 66: ctr - // 67: ap - // 68-75 cr0-7 - // 76: xer - AssignToArrayRange(Builder, Address, Four8, 64, 76); +bool PPC32TargetCodeGenInfo::isStructReturnInRegABI( + const llvm::Triple &Triple, const CodeGenOptions &Opts) { + assert(Triple.getArch() == llvm::Triple::ppc); - // 77-108: v0-31, the 16-byte vector registers - AssignToArrayRange(Builder, Address, Sixteen8, 77, 108); + switch (Opts.getStructReturnConvention()) { + case CodeGenOptions::SRCK_Default: + break; + case CodeGenOptions::SRCK_OnStack: // -maix-struct-return + return false; + case CodeGenOptions::SRCK_InRegs: // -msvr4-struct-return + return true; + } - // 109: vrsave - // 110: vscr - // 111: spe_acc - // 112: spefscr - // 113: sfp - AssignToArrayRange(Builder, Address, Four8, 109, 113); + if (Triple.isOSBinFormatELF() && !Triple.isOSLinux()) + return true; return false; } +bool +PPC32TargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF, + llvm::Value *Address) const { + return PPC_initDwarfEHRegSizeTable(CGF, Address, /*Is64Bit*/ false, + /*IsAIX*/ false); +} + // PowerPC-64 namespace { @@ -4477,8 +4970,8 @@ public: PPC64_SVR4_TargetCodeGenInfo(CodeGenTypes &CGT, PPC64_SVR4_ABIInfo::ABIKind Kind, bool HasQPX, bool SoftFloatABI) - : TargetCodeGenInfo(new PPC64_SVR4_ABIInfo(CGT, Kind, HasQPX, - SoftFloatABI)) {} + : TargetCodeGenInfo(std::make_unique<PPC64_SVR4_ABIInfo>( + CGT, Kind, HasQPX, SoftFloatABI)) {} int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override { // This is recovered from gcc output. @@ -4513,7 +5006,7 @@ PPC64_SVR4_ABIInfo::isPromotableTypeForABI(QualType Ty) const { Ty = EnumTy->getDecl()->getIntegerType(); // Promotable integer types are required to be promoted by the ABI. - if (Ty->isPromotableIntegerType()) + if (isPromotableIntegerTypeForABI(Ty)) return true; // In addition to the usual promotable integer types, we also need to @@ -4527,6 +5020,10 @@ PPC64_SVR4_ABIInfo::isPromotableTypeForABI(QualType Ty) const { break; } + if (const auto *EIT = Ty->getAs<ExtIntType>()) + if (EIT->getNumBits() < 64) + return true; + return false; } @@ -4744,6 +5241,10 @@ PPC64_SVR4_ABIInfo::classifyArgumentType(QualType Ty) const { } } + if (const auto *EIT = Ty->getAs<ExtIntType>()) + if (EIT->getNumBits() > 128) + return getNaturalAlignIndirect(Ty, /*ByVal=*/true); + if (isAggregateTypeForABI(Ty)) { if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory); @@ -4816,6 +5317,10 @@ PPC64_SVR4_ABIInfo::classifyReturnType(QualType RetTy) const { } } + if (const auto *EIT = RetTy->getAs<ExtIntType>()) + if (EIT->getNumBits() > 128) + return getNaturalAlignIndirect(RetTy, /*ByVal=*/false); + if (isAggregateTypeForABI(RetTy)) { // ELFv2 homogeneous aggregates are returned as array types. const Type *Base = nullptr; @@ -4901,66 +5406,19 @@ Address PPC64_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, TypeInfo, SlotSize, /*AllowHigher*/ true); } -static bool -PPC64_initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF, - llvm::Value *Address) { - // This is calculated from the LLVM and GCC tables and verified - // against gcc output. AFAIK all ABIs use the same encoding. - - CodeGen::CGBuilderTy &Builder = CGF.Builder; - - llvm::IntegerType *i8 = CGF.Int8Ty; - llvm::Value *Four8 = llvm::ConstantInt::get(i8, 4); - llvm::Value *Eight8 = llvm::ConstantInt::get(i8, 8); - llvm::Value *Sixteen8 = llvm::ConstantInt::get(i8, 16); - - // 0-31: r0-31, the 8-byte general-purpose registers - AssignToArrayRange(Builder, Address, Eight8, 0, 31); - - // 32-63: fp0-31, the 8-byte floating-point registers - AssignToArrayRange(Builder, Address, Eight8, 32, 63); - - // 64-67 are various 8-byte special-purpose registers: - // 64: mq - // 65: lr - // 66: ctr - // 67: ap - AssignToArrayRange(Builder, Address, Eight8, 64, 67); - - // 68-76 are various 4-byte special-purpose registers: - // 68-75 cr0-7 - // 76: xer - AssignToArrayRange(Builder, Address, Four8, 68, 76); - - // 77-108: v0-31, the 16-byte vector registers - AssignToArrayRange(Builder, Address, Sixteen8, 77, 108); - - // 109: vrsave - // 110: vscr - // 111: spe_acc - // 112: spefscr - // 113: sfp - // 114: tfhar - // 115: tfiar - // 116: texasr - AssignToArrayRange(Builder, Address, Eight8, 109, 116); - - return false; -} - bool PPC64_SVR4_TargetCodeGenInfo::initDwarfEHRegSizeTable( CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const { - - return PPC64_initDwarfEHRegSizeTable(CGF, Address); + return PPC_initDwarfEHRegSizeTable(CGF, Address, /*Is64Bit*/ true, + /*IsAIX*/ false); } bool PPC64TargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const { - - return PPC64_initDwarfEHRegSizeTable(CGF, Address); + return PPC_initDwarfEHRegSizeTable(CGF, Address, /*Is64Bit*/ true, + /*IsAIX*/ false); } //===----------------------------------------------------------------------===// @@ -5031,12 +5489,16 @@ private: bool isLegalVectorTypeForSwift(CharUnits totalSize, llvm::Type *eltTy, unsigned elts) const override; + + bool allowBFloatArgsAndRet() const override { + return getTarget().hasBFloat16Type(); + } }; class AArch64TargetCodeGenInfo : public TargetCodeGenInfo { public: AArch64TargetCodeGenInfo(CodeGenTypes &CGT, AArch64ABIInfo::ABIKind Kind) - : TargetCodeGenInfo(new AArch64ABIInfo(CGT, Kind)) {} + : TargetCodeGenInfo(std::make_unique<AArch64ABIInfo>(CGT, Kind)) {} StringRef getARCRetainAutoreleasedReturnValueMarker() const override { return "mov\tfp, fp\t\t// marker for objc_retainAutoreleaseReturnValue"; @@ -5054,9 +5516,11 @@ public: if (!FD) return; - CodeGenOptions::SignReturnAddressScope Scope = CGM.getCodeGenOpts().getSignReturnAddress(); - CodeGenOptions::SignReturnAddressKeyValue Key = CGM.getCodeGenOpts().getSignReturnAddressKey(); - bool BranchTargetEnforcement = CGM.getCodeGenOpts().BranchTargetEnforcement; + LangOptions::SignReturnAddressScopeKind Scope = + CGM.getLangOpts().getSignReturnAddressScope(); + LangOptions::SignReturnAddressKeyKind Key = + CGM.getLangOpts().getSignReturnAddressKey(); + bool BranchTargetEnforcement = CGM.getLangOpts().BranchTargetEnforcement; if (const auto *TA = FD->getAttr<TargetAttr>()) { ParsedTargetAttr Attr = TA->parse(); if (!Attr.BranchProtection.empty()) { @@ -5072,14 +5536,14 @@ public: } auto *Fn = cast<llvm::Function>(GV); - if (Scope != CodeGenOptions::SignReturnAddressScope::None) { + if (Scope != LangOptions::SignReturnAddressScopeKind::None) { Fn->addFnAttr("sign-return-address", - Scope == CodeGenOptions::SignReturnAddressScope::All + Scope == LangOptions::SignReturnAddressScopeKind::All ? "all" : "non-leaf"); Fn->addFnAttr("sign-return-address-key", - Key == CodeGenOptions::SignReturnAddressKeyValue::AKey + Key == LangOptions::SignReturnAddressKeyKind::AKey ? "a_key" : "b_key"); } @@ -5133,13 +5597,13 @@ ABIArgInfo AArch64ABIInfo::classifyArgumentType(QualType Ty) const { return ABIArgInfo::getDirect(ResType); } if (Size == 64) { - llvm::Type *ResType = - llvm::VectorType::get(llvm::Type::getInt32Ty(getVMContext()), 2); + auto *ResType = + llvm::FixedVectorType::get(llvm::Type::getInt32Ty(getVMContext()), 2); return ABIArgInfo::getDirect(ResType); } if (Size == 128) { - llvm::Type *ResType = - llvm::VectorType::get(llvm::Type::getInt32Ty(getVMContext()), 4); + auto *ResType = + llvm::FixedVectorType::get(llvm::Type::getInt32Ty(getVMContext()), 4); return ABIArgInfo::getDirect(ResType); } return getNaturalAlignIndirect(Ty, /*ByVal=*/false); @@ -5150,7 +5614,11 @@ ABIArgInfo AArch64ABIInfo::classifyArgumentType(QualType Ty) const { if (const EnumType *EnumTy = Ty->getAs<EnumType>()) Ty = EnumTy->getDecl()->getIntegerType(); - return (Ty->isPromotableIntegerType() && isDarwinPCS() + if (const auto *EIT = Ty->getAs<ExtIntType>()) + if (EIT->getNumBits() > 128) + return getNaturalAlignIndirect(Ty); + + return (isPromotableIntegerTypeForABI(Ty) && isDarwinPCS() ? ABIArgInfo::getExtend(Ty) : ABIArgInfo::getDirect()); } @@ -5227,7 +5695,11 @@ ABIArgInfo AArch64ABIInfo::classifyReturnType(QualType RetTy, if (const EnumType *EnumTy = RetTy->getAs<EnumType>()) RetTy = EnumTy->getDecl()->getIntegerType(); - return (RetTy->isPromotableIntegerType() && isDarwinPCS() + if (const auto *EIT = RetTy->getAs<ExtIntType>()) + if (EIT->getNumBits() > 128) + return getNaturalAlignIndirect(RetTy); + + return (isPromotableIntegerTypeForABI(RetTy) && isDarwinPCS() ? ABIArgInfo::getExtend(RetTy) : ABIArgInfo::getDirect()); } @@ -5626,11 +6098,14 @@ public: private: ABIKind Kind; + bool IsFloatABISoftFP; public: ARMABIInfo(CodeGenTypes &CGT, ABIKind _Kind) : SwiftABIInfo(CGT), Kind(_Kind) { setCCs(); + IsFloatABISoftFP = CGT.getCodeGenOpts().FloatABI == "softfp" || + CGT.getCodeGenOpts().FloatABI == ""; // default } bool isEABI() const { @@ -5661,6 +6136,10 @@ public: ABIKind getABIKind() const { return Kind; } + bool allowBFloatArgsAndRet() const override { + return !IsFloatABISoftFP && getTarget().hasBFloat16Type(); + } + private: ABIArgInfo classifyReturnType(QualType RetTy, bool isVariadic, unsigned functionCallConv) const; @@ -5701,7 +6180,7 @@ private: class ARMTargetCodeGenInfo : public TargetCodeGenInfo { public: ARMTargetCodeGenInfo(CodeGenTypes &CGT, ARMABIInfo::ABIKind K) - :TargetCodeGenInfo(new ARMABIInfo(CGT, K)) {} + : TargetCodeGenInfo(std::make_unique<ARMABIInfo>(CGT, K)) {} const ARMABIInfo &getABIInfo() const { return static_cast<const ARMABIInfo&>(TargetCodeGenInfo::getABIInfo()); @@ -5856,7 +6335,7 @@ ABIArgInfo ARMABIInfo::coerceIllegalVector(QualType Ty) const { return ABIArgInfo::getDirect(ResType); } if (Size == 64 || Size == 128) { - llvm::Type *ResType = llvm::VectorType::get( + auto *ResType = llvm::FixedVectorType::get( llvm::Type::getInt32Ty(getVMContext()), Size / 32); return ABIArgInfo::getDirect(ResType); } @@ -5872,7 +6351,7 @@ ABIArgInfo ARMABIInfo::classifyHomogeneousAggregate(QualType Ty, // FP16 vectors should be converted to integer vectors if (!getTarget().hasLegalHalfType() && containsAnyFP16Vectors(Ty)) { uint64_t Size = getContext().getTypeSize(VT); - llvm::Type *NewVecTy = llvm::VectorType::get( + auto *NewVecTy = llvm::FixedVectorType::get( llvm::Type::getInt32Ty(getVMContext()), Size / 32); llvm::Type *Ty = llvm::ArrayType::get(NewVecTy, Members); return ABIArgInfo::getDirect(Ty, 0, nullptr, false); @@ -5900,25 +6379,18 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, bool isVariadic, if (isIllegalVectorType(Ty)) return coerceIllegalVector(Ty); - // _Float16 and __fp16 get passed as if it were an int or float, but with - // the top 16 bits unspecified. This is not done for OpenCL as it handles the - // half type natively, and does not need to interwork with AAPCS code. - if ((Ty->isFloat16Type() || Ty->isHalfType()) && - !getContext().getLangOpts().NativeHalfArgsAndReturns) { - llvm::Type *ResType = IsAAPCS_VFP ? - llvm::Type::getFloatTy(getVMContext()) : - llvm::Type::getInt32Ty(getVMContext()); - return ABIArgInfo::getDirect(ResType); - } - if (!isAggregateTypeForABI(Ty)) { // Treat an enum type as its underlying type. if (const EnumType *EnumTy = Ty->getAs<EnumType>()) { Ty = EnumTy->getDecl()->getIntegerType(); } - return (Ty->isPromotableIntegerType() ? ABIArgInfo::getExtend(Ty) - : ABIArgInfo::getDirect()); + if (const auto *EIT = Ty->getAs<ExtIntType>()) + if (EIT->getNumBits() > 64) + return getNaturalAlignIndirect(Ty, /*ByVal=*/true); + + return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty) + : ABIArgInfo::getDirect()); } if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) { @@ -6100,31 +6572,27 @@ ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy, bool isVariadic, // Large vector types should be returned via memory. if (getContext().getTypeSize(RetTy) > 128) return getNaturalAlignIndirect(RetTy); - // FP16 vectors should be converted to integer vectors - if (!getTarget().hasLegalHalfType() && + // TODO: FP16/BF16 vectors should be converted to integer vectors + // This check is similar to isIllegalVectorType - refactor? + if ((!getTarget().hasLegalHalfType() && (VT->getElementType()->isFloat16Type() || - VT->getElementType()->isHalfType())) + VT->getElementType()->isHalfType())) || + (IsFloatABISoftFP && + VT->getElementType()->isBFloat16Type())) return coerceIllegalVector(RetTy); } - // _Float16 and __fp16 get returned as if it were an int or float, but with - // the top 16 bits unspecified. This is not done for OpenCL as it handles the - // half type natively, and does not need to interwork with AAPCS code. - if ((RetTy->isFloat16Type() || RetTy->isHalfType()) && - !getContext().getLangOpts().NativeHalfArgsAndReturns) { - llvm::Type *ResType = IsAAPCS_VFP ? - llvm::Type::getFloatTy(getVMContext()) : - llvm::Type::getInt32Ty(getVMContext()); - return ABIArgInfo::getDirect(ResType); - } - if (!isAggregateTypeForABI(RetTy)) { // Treat an enum type as its underlying type. if (const EnumType *EnumTy = RetTy->getAs<EnumType>()) RetTy = EnumTy->getDecl()->getIntegerType(); - return RetTy->isPromotableIntegerType() ? ABIArgInfo::getExtend(RetTy) - : ABIArgInfo::getDirect(); + if (const auto *EIT = RetTy->getAs<ExtIntType>()) + if (EIT->getNumBits() > 64) + return getNaturalAlignIndirect(RetTy, /*ByVal=*/false); + + return isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy) + : ABIArgInfo::getDirect(); } // Are we following APCS? @@ -6200,12 +6668,17 @@ ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy, bool isVariadic, /// isIllegalVector - check whether Ty is an illegal vector type. bool ARMABIInfo::isIllegalVectorType(QualType Ty) const { if (const VectorType *VT = Ty->getAs<VectorType> ()) { - // On targets that don't support FP16, FP16 is expanded into float, and we - // don't want the ABI to depend on whether or not FP16 is supported in - // hardware. Thus return false to coerce FP16 vectors into integer vectors. - if (!getTarget().hasLegalHalfType() && + // On targets that don't support half, fp16 or bfloat, they are expanded + // into float, and we don't want the ABI to depend on whether or not they + // are supported in hardware. Thus return false to coerce vectors of these + // types into integer vectors. + // We do not depend on hasLegalHalfType for bfloat as it is a + // separate IR type. + if ((!getTarget().hasLegalHalfType() && (VT->getElementType()->isFloat16Type() || - VT->getElementType()->isHalfType())) + VT->getElementType()->isHalfType())) || + (IsFloatABISoftFP && + VT->getElementType()->isBFloat16Type())) return true; if (isAndroid()) { // Android shipped using Clang 3.1, which supported a slightly different @@ -6257,6 +6730,7 @@ bool ARMABIInfo::containsAnyFP16Vectors(QualType Ty) const { } else { if (const VectorType *VT = Ty->getAs<VectorType>()) return (VT->getElementType()->isFloat16Type() || + VT->getElementType()->isBFloat16Type() || VT->getElementType()->isHalfType()); return false; } @@ -6362,9 +6836,14 @@ Address ARMABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, namespace { +class NVPTXTargetCodeGenInfo; + class NVPTXABIInfo : public ABIInfo { + NVPTXTargetCodeGenInfo &CGInfo; + public: - NVPTXABIInfo(CodeGenTypes &CGT) : ABIInfo(CGT) {} + NVPTXABIInfo(CodeGenTypes &CGT, NVPTXTargetCodeGenInfo &Info) + : ABIInfo(CGT), CGInfo(Info) {} ABIArgInfo classifyReturnType(QualType RetTy) const; ABIArgInfo classifyArgumentType(QualType Ty) const; @@ -6372,36 +6851,87 @@ public: void computeInfo(CGFunctionInfo &FI) const override; Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty) const override; + bool isUnsupportedType(QualType T) const; + ABIArgInfo coerceToIntArrayWithLimit(QualType Ty, unsigned MaxSize) const; }; class NVPTXTargetCodeGenInfo : public TargetCodeGenInfo { public: NVPTXTargetCodeGenInfo(CodeGenTypes &CGT) - : TargetCodeGenInfo(new NVPTXABIInfo(CGT)) {} + : TargetCodeGenInfo(std::make_unique<NVPTXABIInfo>(CGT, *this)) {} void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const override; bool shouldEmitStaticExternCAliases() const override; + llvm::Type *getCUDADeviceBuiltinSurfaceDeviceType() const override { + // On the device side, surface reference is represented as an object handle + // in 64-bit integer. + return llvm::Type::getInt64Ty(getABIInfo().getVMContext()); + } + + llvm::Type *getCUDADeviceBuiltinTextureDeviceType() const override { + // On the device side, texture reference is represented as an object handle + // in 64-bit integer. + return llvm::Type::getInt64Ty(getABIInfo().getVMContext()); + } + + bool emitCUDADeviceBuiltinSurfaceDeviceCopy(CodeGenFunction &CGF, LValue Dst, + LValue Src) const override { + emitBuiltinSurfTexDeviceCopy(CGF, Dst, Src); + return true; + } + + bool emitCUDADeviceBuiltinTextureDeviceCopy(CodeGenFunction &CGF, LValue Dst, + LValue Src) const override { + emitBuiltinSurfTexDeviceCopy(CGF, Dst, Src); + return true; + } + private: - // Adds a NamedMDNode with F, Name, and Operand as operands, and adds the + // Adds a NamedMDNode with GV, Name, and Operand as operands, and adds the // resulting MDNode to the nvvm.annotations MDNode. - static void addNVVMMetadata(llvm::Function *F, StringRef Name, int Operand); + static void addNVVMMetadata(llvm::GlobalValue *GV, StringRef Name, + int Operand); + + static void emitBuiltinSurfTexDeviceCopy(CodeGenFunction &CGF, LValue Dst, + LValue Src) { + llvm::Value *Handle = nullptr; + llvm::Constant *C = + llvm::dyn_cast<llvm::Constant>(Src.getAddress(CGF).getPointer()); + // Lookup `addrspacecast` through the constant pointer if any. + if (auto *ASC = llvm::dyn_cast_or_null<llvm::AddrSpaceCastOperator>(C)) + C = llvm::cast<llvm::Constant>(ASC->getPointerOperand()); + if (auto *GV = llvm::dyn_cast_or_null<llvm::GlobalVariable>(C)) { + // Load the handle from the specific global variable using + // `nvvm.texsurf.handle.internal` intrinsic. + Handle = CGF.EmitRuntimeCall( + CGF.CGM.getIntrinsic(llvm::Intrinsic::nvvm_texsurf_handle_internal, + {GV->getType()}), + {GV}, "texsurf_handle"); + } else + Handle = CGF.EmitLoadOfScalar(Src, SourceLocation()); + CGF.EmitStoreOfScalar(Handle, Dst); + } }; /// Checks if the type is unsupported directly by the current target. -static bool isUnsupportedType(ASTContext &Context, QualType T) { +bool NVPTXABIInfo::isUnsupportedType(QualType T) const { + ASTContext &Context = getContext(); if (!Context.getTargetInfo().hasFloat16Type() && T->isFloat16Type()) return true; if (!Context.getTargetInfo().hasFloat128Type() && (T->isFloat128Type() || (T->isRealFloatingType() && Context.getTypeSize(T) == 128))) return true; + if (const auto *EIT = T->getAs<ExtIntType>()) + return EIT->getNumBits() > + (Context.getTargetInfo().hasInt128Type() ? 128U : 64U); if (!Context.getTargetInfo().hasInt128Type() && T->isIntegerType() && - Context.getTypeSize(T) > 64) + Context.getTypeSize(T) > 64U) return true; if (const auto *AT = T->getAsArrayTypeUnsafe()) - return isUnsupportedType(Context, AT->getElementType()); + return isUnsupportedType(AT->getElementType()); const auto *RT = T->getAs<RecordType>(); if (!RT) return false; @@ -6410,24 +6940,23 @@ static bool isUnsupportedType(ASTContext &Context, QualType T) { // If this is a C++ record, check the bases first. if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) for (const CXXBaseSpecifier &I : CXXRD->bases()) - if (isUnsupportedType(Context, I.getType())) + if (isUnsupportedType(I.getType())) return true; for (const FieldDecl *I : RD->fields()) - if (isUnsupportedType(Context, I->getType())) + if (isUnsupportedType(I->getType())) return true; return false; } /// Coerce the given type into an array with maximum allowed size of elements. -static ABIArgInfo coerceToIntArrayWithLimit(QualType Ty, ASTContext &Context, - llvm::LLVMContext &LLVMContext, - unsigned MaxSize) { +ABIArgInfo NVPTXABIInfo::coerceToIntArrayWithLimit(QualType Ty, + unsigned MaxSize) const { // Alignment and Size are measured in bits. - const uint64_t Size = Context.getTypeSize(Ty); - const uint64_t Alignment = Context.getTypeAlign(Ty); + const uint64_t Size = getContext().getTypeSize(Ty); + const uint64_t Alignment = getContext().getTypeAlign(Ty); const unsigned Div = std::min<unsigned>(MaxSize, Alignment); - llvm::Type *IntType = llvm::Type::getIntNTy(LLVMContext, Div); + llvm::Type *IntType = llvm::Type::getIntNTy(getVMContext(), Div); const uint64_t NumElements = (Size + Div - 1) / Div; return ABIArgInfo::getDirect(llvm::ArrayType::get(IntType, NumElements)); } @@ -6437,9 +6966,8 @@ ABIArgInfo NVPTXABIInfo::classifyReturnType(QualType RetTy) const { return ABIArgInfo::getIgnore(); if (getContext().getLangOpts().OpenMP && - getContext().getLangOpts().OpenMPIsDevice && - isUnsupportedType(getContext(), RetTy)) - return coerceToIntArrayWithLimit(RetTy, getContext(), getVMContext(), 64); + getContext().getLangOpts().OpenMPIsDevice && isUnsupportedType(RetTy)) + return coerceToIntArrayWithLimit(RetTy, 64); // note: this is different from default ABI if (!RetTy->isScalarType()) @@ -6449,8 +6977,8 @@ ABIArgInfo NVPTXABIInfo::classifyReturnType(QualType RetTy) const { if (const EnumType *EnumTy = RetTy->getAs<EnumType>()) RetTy = EnumTy->getDecl()->getIntegerType(); - return (RetTy->isPromotableIntegerType() ? ABIArgInfo::getExtend(RetTy) - : ABIArgInfo::getDirect()); + return (isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy) + : ABIArgInfo::getDirect()); } ABIArgInfo NVPTXABIInfo::classifyArgumentType(QualType Ty) const { @@ -6459,11 +6987,29 @@ ABIArgInfo NVPTXABIInfo::classifyArgumentType(QualType Ty) const { Ty = EnumTy->getDecl()->getIntegerType(); // Return aggregates type as indirect by value - if (isAggregateTypeForABI(Ty)) + if (isAggregateTypeForABI(Ty)) { + // Under CUDA device compilation, tex/surf builtin types are replaced with + // object types and passed directly. + if (getContext().getLangOpts().CUDAIsDevice) { + if (Ty->isCUDADeviceBuiltinSurfaceType()) + return ABIArgInfo::getDirect( + CGInfo.getCUDADeviceBuiltinSurfaceDeviceType()); + if (Ty->isCUDADeviceBuiltinTextureType()) + return ABIArgInfo::getDirect( + CGInfo.getCUDADeviceBuiltinTextureDeviceType()); + } return getNaturalAlignIndirect(Ty, /* byval */ true); + } - return (Ty->isPromotableIntegerType() ? ABIArgInfo::getExtend(Ty) - : ABIArgInfo::getDirect()); + if (const auto *EIT = Ty->getAs<ExtIntType>()) { + if ((EIT->getNumBits() > 128) || + (!getContext().getTargetInfo().hasInt128Type() && + EIT->getNumBits() > 64)) + return getNaturalAlignIndirect(Ty, /* byval */ true); + } + + return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty) + : ABIArgInfo::getDirect()); } void NVPTXABIInfo::computeInfo(CGFunctionInfo &FI) const { @@ -6488,6 +7034,17 @@ void NVPTXTargetCodeGenInfo::setTargetAttributes( const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const { if (GV->isDeclaration()) return; + const VarDecl *VD = dyn_cast_or_null<VarDecl>(D); + if (VD) { + if (M.getLangOpts().CUDA) { + if (VD->getType()->isCUDADeviceBuiltinSurfaceType()) + addNVVMMetadata(GV, "surface", 1); + else if (VD->getType()->isCUDADeviceBuiltinTextureType()) + addNVVMMetadata(GV, "texture", 1); + return; + } + } + const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D); if (!FD) return; @@ -6536,16 +7093,16 @@ void NVPTXTargetCodeGenInfo::setTargetAttributes( } } -void NVPTXTargetCodeGenInfo::addNVVMMetadata(llvm::Function *F, StringRef Name, - int Operand) { - llvm::Module *M = F->getParent(); +void NVPTXTargetCodeGenInfo::addNVVMMetadata(llvm::GlobalValue *GV, + StringRef Name, int Operand) { + llvm::Module *M = GV->getParent(); llvm::LLVMContext &Ctx = M->getContext(); // Get "nvvm.annotations" metadata node llvm::NamedMDNode *MD = M->getOrInsertNamedMetadata("nvvm.annotations"); llvm::Metadata *MDVals[] = { - llvm::ConstantAsMetadata::get(F), llvm::MDString::get(Ctx, Name), + llvm::ConstantAsMetadata::get(GV), llvm::MDString::get(Ctx, Name), llvm::ConstantAsMetadata::get( llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), Operand))}; // Append metadata to nvvm.annotations @@ -6565,12 +7122,13 @@ namespace { class SystemZABIInfo : public SwiftABIInfo { bool HasVector; + bool IsSoftFloatABI; public: - SystemZABIInfo(CodeGenTypes &CGT, bool HV) - : SwiftABIInfo(CGT), HasVector(HV) {} + SystemZABIInfo(CodeGenTypes &CGT, bool HV, bool SF) + : SwiftABIInfo(CGT), HasVector(HV), IsSoftFloatABI(SF) {} - bool isPromotableIntegerType(QualType Ty) const; + bool isPromotableIntegerTypeForABI(QualType Ty) const; bool isCompoundType(QualType Ty) const; bool isVectorArgumentType(QualType Ty) const; bool isFPArgumentType(QualType Ty) const; @@ -6600,21 +7158,26 @@ public: class SystemZTargetCodeGenInfo : public TargetCodeGenInfo { public: - SystemZTargetCodeGenInfo(CodeGenTypes &CGT, bool HasVector) - : TargetCodeGenInfo(new SystemZABIInfo(CGT, HasVector)) {} + SystemZTargetCodeGenInfo(CodeGenTypes &CGT, bool HasVector, bool SoftFloatABI) + : TargetCodeGenInfo( + std::make_unique<SystemZABIInfo>(CGT, HasVector, SoftFloatABI)) {} }; } -bool SystemZABIInfo::isPromotableIntegerType(QualType Ty) const { +bool SystemZABIInfo::isPromotableIntegerTypeForABI(QualType Ty) const { // Treat an enum type as its underlying type. if (const EnumType *EnumTy = Ty->getAs<EnumType>()) Ty = EnumTy->getDecl()->getIntegerType(); // Promotable integer types are required to be promoted by the ABI. - if (Ty->isPromotableIntegerType()) + if (ABIInfo::isPromotableIntegerTypeForABI(Ty)) return true; + if (const auto *EIT = Ty->getAs<ExtIntType>()) + if (EIT->getNumBits() < 64) + return true; + // 32-bit values must also be promoted. if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) switch (BT->getKind()) { @@ -6640,6 +7203,9 @@ bool SystemZABIInfo::isVectorArgumentType(QualType Ty) const { } bool SystemZABIInfo::isFPArgumentType(QualType Ty) const { + if (IsSoftFloatABI) + return false; + if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) switch (BT->getKind()) { case BuiltinType::Float: @@ -6653,7 +7219,9 @@ bool SystemZABIInfo::isFPArgumentType(QualType Ty) const { } QualType SystemZABIInfo::GetSingleElementType(QualType Ty) const { - if (const RecordType *RT = Ty->getAsStructureType()) { + const RecordType *RT = Ty->getAs<RecordType>(); + + if (RT && RT->isStructureOrClassType()) { const RecordDecl *RD = RT->getDecl(); QualType Found; @@ -6679,6 +7247,10 @@ QualType SystemZABIInfo::GetSingleElementType(QualType Ty) const { if (getContext().getLangOpts().CPlusPlus && FD->isZeroLengthBitField(getContext())) continue; + // Like isSingleElementStruct(), ignore C++20 empty data members. + if (FD->hasAttr<NoUniqueAddressAttr>() && + isEmptyRecord(getContext(), FD->getType(), true)) + continue; // Unlike isSingleElementStruct(), arrays do not count. // Nested structures still do though. @@ -6725,7 +7297,7 @@ Address SystemZABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, } else { if (AI.getCoerceToType()) ArgTy = AI.getCoerceToType(); - InFPRs = ArgTy->isFloatTy() || ArgTy->isDoubleTy(); + InFPRs = (!IsSoftFloatABI && (ArgTy->isFloatTy() || ArgTy->isDoubleTy())); IsVector = ArgTy->isVectorTy(); UnpaddedSize = TyInfo.first; DirectAlign = TyInfo.second; @@ -6858,8 +7430,8 @@ ABIArgInfo SystemZABIInfo::classifyReturnType(QualType RetTy) const { return ABIArgInfo::getDirect(); if (isCompoundType(RetTy) || getContext().getTypeSize(RetTy) > 64) return getNaturalAlignIndirect(RetTy); - return (isPromotableIntegerType(RetTy) ? ABIArgInfo::getExtend(RetTy) - : ABIArgInfo::getDirect()); + return (isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy) + : ABIArgInfo::getDirect()); } ABIArgInfo SystemZABIInfo::classifyArgumentType(QualType Ty) const { @@ -6868,7 +7440,7 @@ ABIArgInfo SystemZABIInfo::classifyArgumentType(QualType Ty) const { return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory); // Integers and enums are extended to full register width. - if (isPromotableIntegerType(Ty)) + if (isPromotableIntegerTypeForABI(Ty)) return ABIArgInfo::getExtend(Ty); // Handle vector types and vector-like structure types. Note that @@ -6918,10 +7490,49 @@ ABIArgInfo SystemZABIInfo::classifyArgumentType(QualType Ty) const { namespace { +class MSP430ABIInfo : public DefaultABIInfo { + static ABIArgInfo complexArgInfo() { + ABIArgInfo Info = ABIArgInfo::getDirect(); + Info.setCanBeFlattened(false); + return Info; + } + +public: + MSP430ABIInfo(CodeGenTypes &CGT) : DefaultABIInfo(CGT) {} + + ABIArgInfo classifyReturnType(QualType RetTy) const { + if (RetTy->isAnyComplexType()) + return complexArgInfo(); + + return DefaultABIInfo::classifyReturnType(RetTy); + } + + ABIArgInfo classifyArgumentType(QualType RetTy) const { + if (RetTy->isAnyComplexType()) + return complexArgInfo(); + + return DefaultABIInfo::classifyArgumentType(RetTy); + } + + // Just copy the original implementations because + // DefaultABIInfo::classify{Return,Argument}Type() are not virtual + void computeInfo(CGFunctionInfo &FI) const override { + if (!getCXXABI().classifyReturnType(FI)) + FI.getReturnInfo() = classifyReturnType(FI.getReturnType()); + for (auto &I : FI.arguments()) + I.info = classifyArgumentType(I.type); + } + + Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const override { + return EmitVAArgInstr(CGF, VAListAddr, Ty, classifyArgumentType(Ty)); + } +}; + class MSP430TargetCodeGenInfo : public TargetCodeGenInfo { public: MSP430TargetCodeGenInfo(CodeGenTypes &CGT) - : TargetCodeGenInfo(new DefaultABIInfo(CGT)) {} + : TargetCodeGenInfo(std::make_unique<MSP430ABIInfo>(CGT)) {} void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const override; }; @@ -6980,8 +7591,8 @@ class MIPSTargetCodeGenInfo : public TargetCodeGenInfo { unsigned SizeOfUnwindException; public: MIPSTargetCodeGenInfo(CodeGenTypes &CGT, bool IsO32) - : TargetCodeGenInfo(new MipsABIInfo(CGT, IsO32)), - SizeOfUnwindException(IsO32 ? 24 : 32) {} + : TargetCodeGenInfo(std::make_unique<MipsABIInfo>(CGT, IsO32)), + SizeOfUnwindException(IsO32 ? 24 : 32) {} int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override { return 29; @@ -7163,6 +7774,13 @@ MipsABIInfo::classifyArgumentType(QualType Ty, uint64_t &Offset) const { if (const EnumType *EnumTy = Ty->getAs<EnumType>()) Ty = EnumTy->getDecl()->getIntegerType(); + // Make sure we pass indirectly things that are too large. + if (const auto *EIT = Ty->getAs<ExtIntType>()) + if (EIT->getNumBits() > 128 || + (EIT->getNumBits() > 64 && + !getContext().getTargetInfo().hasInt128Type())) + return getNaturalAlignIndirect(Ty); + // All integral types are promoted to the GPR width. if (Ty->isIntegralOrEnumerationType()) return extendType(Ty); @@ -7247,7 +7865,14 @@ ABIArgInfo MipsABIInfo::classifyReturnType(QualType RetTy) const { if (const EnumType *EnumTy = RetTy->getAs<EnumType>()) RetTy = EnumTy->getDecl()->getIntegerType(); - if (RetTy->isPromotableIntegerType()) + // Make sure we pass indirectly things that are too large. + if (const auto *EIT = RetTy->getAs<ExtIntType>()) + if (EIT->getNumBits() > 128 || + (EIT->getNumBits() > 64 && + !getContext().getTargetInfo().hasInt128Type())) + return getNaturalAlignIndirect(RetTy); + + if (isPromotableIntegerTypeForABI(RetTy)) return ABIArgInfo::getExtend(RetTy); if ((RetTy->isUnsignedIntegerOrEnumerationType() || @@ -7366,7 +7991,7 @@ namespace { class AVRTargetCodeGenInfo : public TargetCodeGenInfo { public: AVRTargetCodeGenInfo(CodeGenTypes &CGT) - : TargetCodeGenInfo(new DefaultABIInfo(CGT)) { } + : TargetCodeGenInfo(std::make_unique<DefaultABIInfo>(CGT)) {} void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const override { @@ -7455,50 +8080,97 @@ void TCETargetCodeGenInfo::setTargetAttributes( namespace { -class HexagonABIInfo : public ABIInfo { - - +class HexagonABIInfo : public DefaultABIInfo { public: - HexagonABIInfo(CodeGenTypes &CGT) : ABIInfo(CGT) {} + HexagonABIInfo(CodeGenTypes &CGT) : DefaultABIInfo(CGT) {} private: - ABIArgInfo classifyReturnType(QualType RetTy) const; ABIArgInfo classifyArgumentType(QualType RetTy) const; + ABIArgInfo classifyArgumentType(QualType RetTy, unsigned *RegsLeft) const; void computeInfo(CGFunctionInfo &FI) const override; Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty) const override; + Address EmitVAArgFromMemory(CodeGenFunction &CFG, Address VAListAddr, + QualType Ty) const; + Address EmitVAArgForHexagon(CodeGenFunction &CFG, Address VAListAddr, + QualType Ty) const; + Address EmitVAArgForHexagonLinux(CodeGenFunction &CFG, Address VAListAddr, + QualType Ty) const; }; class HexagonTargetCodeGenInfo : public TargetCodeGenInfo { public: HexagonTargetCodeGenInfo(CodeGenTypes &CGT) - :TargetCodeGenInfo(new HexagonABIInfo(CGT)) {} + : TargetCodeGenInfo(std::make_unique<HexagonABIInfo>(CGT)) {} int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override { return 29; } + + void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, + CodeGen::CodeGenModule &GCM) const override { + if (GV->isDeclaration()) + return; + const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D); + if (!FD) + return; + } }; -} +} // namespace void HexagonABIInfo::computeInfo(CGFunctionInfo &FI) const { + unsigned RegsLeft = 6; if (!getCXXABI().classifyReturnType(FI)) FI.getReturnInfo() = classifyReturnType(FI.getReturnType()); for (auto &I : FI.arguments()) - I.info = classifyArgumentType(I.type); + I.info = classifyArgumentType(I.type, &RegsLeft); } -ABIArgInfo HexagonABIInfo::classifyArgumentType(QualType Ty) const { +static bool HexagonAdjustRegsLeft(uint64_t Size, unsigned *RegsLeft) { + assert(Size <= 64 && "Not expecting to pass arguments larger than 64 bits" + " through registers"); + + if (*RegsLeft == 0) + return false; + + if (Size <= 32) { + (*RegsLeft)--; + return true; + } + + if (2 <= (*RegsLeft & (~1U))) { + *RegsLeft = (*RegsLeft & (~1U)) - 2; + return true; + } + + // Next available register was r5 but candidate was greater than 32-bits so it + // has to go on the stack. However we still consume r5 + if (*RegsLeft == 1) + *RegsLeft = 0; + + return false; +} + +ABIArgInfo HexagonABIInfo::classifyArgumentType(QualType Ty, + unsigned *RegsLeft) const { if (!isAggregateTypeForABI(Ty)) { // Treat an enum type as its underlying type. if (const EnumType *EnumTy = Ty->getAs<EnumType>()) Ty = EnumTy->getDecl()->getIntegerType(); - return (Ty->isPromotableIntegerType() ? ABIArgInfo::getExtend(Ty) - : ABIArgInfo::getDirect()); + uint64_t Size = getContext().getTypeSize(Ty); + if (Size <= 64) + HexagonAdjustRegsLeft(Size, RegsLeft); + + if (Size > 64 && Ty->isExtIntType()) + return getNaturalAlignIndirect(Ty, /*ByVal=*/true); + + return isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty) + : ABIArgInfo::getDirect(); } if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) @@ -7509,63 +8181,304 @@ ABIArgInfo HexagonABIInfo::classifyArgumentType(QualType Ty) const { return ABIArgInfo::getIgnore(); uint64_t Size = getContext().getTypeSize(Ty); + unsigned Align = getContext().getTypeAlign(Ty); + if (Size > 64) return getNaturalAlignIndirect(Ty, /*ByVal=*/true); + + if (HexagonAdjustRegsLeft(Size, RegsLeft)) + Align = Size <= 32 ? 32 : 64; + if (Size <= Align) { // Pass in the smallest viable integer type. - else if (Size > 32) - return ABIArgInfo::getDirect(llvm::Type::getInt64Ty(getVMContext())); - else if (Size > 16) - return ABIArgInfo::getDirect(llvm::Type::getInt32Ty(getVMContext())); - else if (Size > 8) - return ABIArgInfo::getDirect(llvm::Type::getInt16Ty(getVMContext())); - else - return ABIArgInfo::getDirect(llvm::Type::getInt8Ty(getVMContext())); + if (!llvm::isPowerOf2_64(Size)) + Size = llvm::NextPowerOf2(Size); + return ABIArgInfo::getDirect(llvm::Type::getIntNTy(getVMContext(), Size)); + } + return DefaultABIInfo::classifyArgumentType(Ty); } ABIArgInfo HexagonABIInfo::classifyReturnType(QualType RetTy) const { if (RetTy->isVoidType()) return ABIArgInfo::getIgnore(); - // Large vector types should be returned via memory. - if (RetTy->isVectorType() && getContext().getTypeSize(RetTy) > 64) - return getNaturalAlignIndirect(RetTy); + const TargetInfo &T = CGT.getTarget(); + uint64_t Size = getContext().getTypeSize(RetTy); + + if (RetTy->getAs<VectorType>()) { + // HVX vectors are returned in vector registers or register pairs. + if (T.hasFeature("hvx")) { + assert(T.hasFeature("hvx-length64b") || T.hasFeature("hvx-length128b")); + uint64_t VecSize = T.hasFeature("hvx-length64b") ? 64*8 : 128*8; + if (Size == VecSize || Size == 2*VecSize) + return ABIArgInfo::getDirectInReg(); + } + // Large vector types should be returned via memory. + if (Size > 64) + return getNaturalAlignIndirect(RetTy); + } if (!isAggregateTypeForABI(RetTy)) { // Treat an enum type as its underlying type. if (const EnumType *EnumTy = RetTy->getAs<EnumType>()) RetTy = EnumTy->getDecl()->getIntegerType(); - return (RetTy->isPromotableIntegerType() ? ABIArgInfo::getExtend(RetTy) - : ABIArgInfo::getDirect()); + if (Size > 64 && RetTy->isExtIntType()) + return getNaturalAlignIndirect(RetTy, /*ByVal=*/false); + + return isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy) + : ABIArgInfo::getDirect(); } if (isEmptyRecord(getContext(), RetTy, true)) return ABIArgInfo::getIgnore(); - // Aggregates <= 8 bytes are returned in r0; other aggregates + // Aggregates <= 8 bytes are returned in registers, other aggregates // are returned indirectly. - uint64_t Size = getContext().getTypeSize(RetTy); if (Size <= 64) { // Return in the smallest viable integer type. - if (Size <= 8) - return ABIArgInfo::getDirect(llvm::Type::getInt8Ty(getVMContext())); - if (Size <= 16) - return ABIArgInfo::getDirect(llvm::Type::getInt16Ty(getVMContext())); - if (Size <= 32) - return ABIArgInfo::getDirect(llvm::Type::getInt32Ty(getVMContext())); - return ABIArgInfo::getDirect(llvm::Type::getInt64Ty(getVMContext())); + if (!llvm::isPowerOf2_64(Size)) + Size = llvm::NextPowerOf2(Size); + return ABIArgInfo::getDirect(llvm::Type::getIntNTy(getVMContext(), Size)); } - return getNaturalAlignIndirect(RetTy, /*ByVal=*/true); } +Address HexagonABIInfo::EmitVAArgFromMemory(CodeGenFunction &CGF, + Address VAListAddr, + QualType Ty) const { + // Load the overflow area pointer. + Address __overflow_area_pointer_p = + CGF.Builder.CreateStructGEP(VAListAddr, 2, "__overflow_area_pointer_p"); + llvm::Value *__overflow_area_pointer = CGF.Builder.CreateLoad( + __overflow_area_pointer_p, "__overflow_area_pointer"); + + uint64_t Align = CGF.getContext().getTypeAlign(Ty) / 8; + if (Align > 4) { + // Alignment should be a power of 2. + assert((Align & (Align - 1)) == 0 && "Alignment is not power of 2!"); + + // overflow_arg_area = (overflow_arg_area + align - 1) & -align; + llvm::Value *Offset = llvm::ConstantInt::get(CGF.Int64Ty, Align - 1); + + // Add offset to the current pointer to access the argument. + __overflow_area_pointer = + CGF.Builder.CreateGEP(__overflow_area_pointer, Offset); + llvm::Value *AsInt = + CGF.Builder.CreatePtrToInt(__overflow_area_pointer, CGF.Int32Ty); + + // Create a mask which should be "AND"ed + // with (overflow_arg_area + align - 1) + llvm::Value *Mask = llvm::ConstantInt::get(CGF.Int32Ty, -(int)Align); + __overflow_area_pointer = CGF.Builder.CreateIntToPtr( + CGF.Builder.CreateAnd(AsInt, Mask), __overflow_area_pointer->getType(), + "__overflow_area_pointer.align"); + } + + // Get the type of the argument from memory and bitcast + // overflow area pointer to the argument type. + llvm::Type *PTy = CGF.ConvertTypeForMem(Ty); + Address AddrTyped = CGF.Builder.CreateBitCast( + Address(__overflow_area_pointer, CharUnits::fromQuantity(Align)), + llvm::PointerType::getUnqual(PTy)); + + // Round up to the minimum stack alignment for varargs which is 4 bytes. + uint64_t Offset = llvm::alignTo(CGF.getContext().getTypeSize(Ty) / 8, 4); + + __overflow_area_pointer = CGF.Builder.CreateGEP( + __overflow_area_pointer, llvm::ConstantInt::get(CGF.Int32Ty, Offset), + "__overflow_area_pointer.next"); + CGF.Builder.CreateStore(__overflow_area_pointer, __overflow_area_pointer_p); + + return AddrTyped; +} + +Address HexagonABIInfo::EmitVAArgForHexagon(CodeGenFunction &CGF, + Address VAListAddr, + QualType Ty) const { + // FIXME: Need to handle alignment + llvm::Type *BP = CGF.Int8PtrTy; + llvm::Type *BPP = CGF.Int8PtrPtrTy; + CGBuilderTy &Builder = CGF.Builder; + Address VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP, "ap"); + llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur"); + // Handle address alignment for type alignment > 32 bits + uint64_t TyAlign = CGF.getContext().getTypeAlign(Ty) / 8; + if (TyAlign > 4) { + assert((TyAlign & (TyAlign - 1)) == 0 && "Alignment is not power of 2!"); + llvm::Value *AddrAsInt = Builder.CreatePtrToInt(Addr, CGF.Int32Ty); + AddrAsInt = Builder.CreateAdd(AddrAsInt, Builder.getInt32(TyAlign - 1)); + AddrAsInt = Builder.CreateAnd(AddrAsInt, Builder.getInt32(~(TyAlign - 1))); + Addr = Builder.CreateIntToPtr(AddrAsInt, BP); + } + llvm::Type *PTy = llvm::PointerType::getUnqual(CGF.ConvertType(Ty)); + Address AddrTyped = Builder.CreateBitCast( + Address(Addr, CharUnits::fromQuantity(TyAlign)), PTy); + + uint64_t Offset = llvm::alignTo(CGF.getContext().getTypeSize(Ty) / 8, 4); + llvm::Value *NextAddr = Builder.CreateGEP( + Addr, llvm::ConstantInt::get(CGF.Int32Ty, Offset), "ap.next"); + Builder.CreateStore(NextAddr, VAListAddrAsBPP); + + return AddrTyped; +} + +Address HexagonABIInfo::EmitVAArgForHexagonLinux(CodeGenFunction &CGF, + Address VAListAddr, + QualType Ty) const { + int ArgSize = CGF.getContext().getTypeSize(Ty) / 8; + + if (ArgSize > 8) + return EmitVAArgFromMemory(CGF, VAListAddr, Ty); + + // Here we have check if the argument is in register area or + // in overflow area. + // If the saved register area pointer + argsize rounded up to alignment > + // saved register area end pointer, argument is in overflow area. + unsigned RegsLeft = 6; + Ty = CGF.getContext().getCanonicalType(Ty); + (void)classifyArgumentType(Ty, &RegsLeft); + + llvm::BasicBlock *MaybeRegBlock = CGF.createBasicBlock("vaarg.maybe_reg"); + llvm::BasicBlock *InRegBlock = CGF.createBasicBlock("vaarg.in_reg"); + llvm::BasicBlock *OnStackBlock = CGF.createBasicBlock("vaarg.on_stack"); + llvm::BasicBlock *ContBlock = CGF.createBasicBlock("vaarg.end"); + + // Get rounded size of the argument.GCC does not allow vararg of + // size < 4 bytes. We follow the same logic here. + ArgSize = (CGF.getContext().getTypeSize(Ty) <= 32) ? 4 : 8; + int ArgAlign = (CGF.getContext().getTypeSize(Ty) <= 32) ? 4 : 8; + + // Argument may be in saved register area + CGF.EmitBlock(MaybeRegBlock); + + // Load the current saved register area pointer. + Address __current_saved_reg_area_pointer_p = CGF.Builder.CreateStructGEP( + VAListAddr, 0, "__current_saved_reg_area_pointer_p"); + llvm::Value *__current_saved_reg_area_pointer = CGF.Builder.CreateLoad( + __current_saved_reg_area_pointer_p, "__current_saved_reg_area_pointer"); + + // Load the saved register area end pointer. + Address __saved_reg_area_end_pointer_p = CGF.Builder.CreateStructGEP( + VAListAddr, 1, "__saved_reg_area_end_pointer_p"); + llvm::Value *__saved_reg_area_end_pointer = CGF.Builder.CreateLoad( + __saved_reg_area_end_pointer_p, "__saved_reg_area_end_pointer"); + + // If the size of argument is > 4 bytes, check if the stack + // location is aligned to 8 bytes + if (ArgAlign > 4) { + + llvm::Value *__current_saved_reg_area_pointer_int = + CGF.Builder.CreatePtrToInt(__current_saved_reg_area_pointer, + CGF.Int32Ty); + + __current_saved_reg_area_pointer_int = CGF.Builder.CreateAdd( + __current_saved_reg_area_pointer_int, + llvm::ConstantInt::get(CGF.Int32Ty, (ArgAlign - 1)), + "align_current_saved_reg_area_pointer"); + + __current_saved_reg_area_pointer_int = + CGF.Builder.CreateAnd(__current_saved_reg_area_pointer_int, + llvm::ConstantInt::get(CGF.Int32Ty, -ArgAlign), + "align_current_saved_reg_area_pointer"); + + __current_saved_reg_area_pointer = + CGF.Builder.CreateIntToPtr(__current_saved_reg_area_pointer_int, + __current_saved_reg_area_pointer->getType(), + "align_current_saved_reg_area_pointer"); + } + + llvm::Value *__new_saved_reg_area_pointer = + CGF.Builder.CreateGEP(__current_saved_reg_area_pointer, + llvm::ConstantInt::get(CGF.Int32Ty, ArgSize), + "__new_saved_reg_area_pointer"); + + llvm::Value *UsingStack = 0; + UsingStack = CGF.Builder.CreateICmpSGT(__new_saved_reg_area_pointer, + __saved_reg_area_end_pointer); + + CGF.Builder.CreateCondBr(UsingStack, OnStackBlock, InRegBlock); + + // Argument in saved register area + // Implement the block where argument is in register saved area + CGF.EmitBlock(InRegBlock); + + llvm::Type *PTy = CGF.ConvertType(Ty); + llvm::Value *__saved_reg_area_p = CGF.Builder.CreateBitCast( + __current_saved_reg_area_pointer, llvm::PointerType::getUnqual(PTy)); + + CGF.Builder.CreateStore(__new_saved_reg_area_pointer, + __current_saved_reg_area_pointer_p); + + CGF.EmitBranch(ContBlock); + + // Argument in overflow area + // Implement the block where the argument is in overflow area. + CGF.EmitBlock(OnStackBlock); + + // Load the overflow area pointer + Address __overflow_area_pointer_p = + CGF.Builder.CreateStructGEP(VAListAddr, 2, "__overflow_area_pointer_p"); + llvm::Value *__overflow_area_pointer = CGF.Builder.CreateLoad( + __overflow_area_pointer_p, "__overflow_area_pointer"); + + // Align the overflow area pointer according to the alignment of the argument + if (ArgAlign > 4) { + llvm::Value *__overflow_area_pointer_int = + CGF.Builder.CreatePtrToInt(__overflow_area_pointer, CGF.Int32Ty); + + __overflow_area_pointer_int = + CGF.Builder.CreateAdd(__overflow_area_pointer_int, + llvm::ConstantInt::get(CGF.Int32Ty, ArgAlign - 1), + "align_overflow_area_pointer"); + + __overflow_area_pointer_int = + CGF.Builder.CreateAnd(__overflow_area_pointer_int, + llvm::ConstantInt::get(CGF.Int32Ty, -ArgAlign), + "align_overflow_area_pointer"); + + __overflow_area_pointer = CGF.Builder.CreateIntToPtr( + __overflow_area_pointer_int, __overflow_area_pointer->getType(), + "align_overflow_area_pointer"); + } + + // Get the pointer for next argument in overflow area and store it + // to overflow area pointer. + llvm::Value *__new_overflow_area_pointer = CGF.Builder.CreateGEP( + __overflow_area_pointer, llvm::ConstantInt::get(CGF.Int32Ty, ArgSize), + "__overflow_area_pointer.next"); + + CGF.Builder.CreateStore(__new_overflow_area_pointer, + __overflow_area_pointer_p); + + CGF.Builder.CreateStore(__new_overflow_area_pointer, + __current_saved_reg_area_pointer_p); + + // Bitcast the overflow area pointer to the type of argument. + llvm::Type *OverflowPTy = CGF.ConvertTypeForMem(Ty); + llvm::Value *__overflow_area_p = CGF.Builder.CreateBitCast( + __overflow_area_pointer, llvm::PointerType::getUnqual(OverflowPTy)); + + CGF.EmitBranch(ContBlock); + + // Get the correct pointer to load the variable argument + // Implement the ContBlock + CGF.EmitBlock(ContBlock); + + llvm::Type *MemPTy = llvm::PointerType::getUnqual(CGF.ConvertTypeForMem(Ty)); + llvm::PHINode *ArgAddr = CGF.Builder.CreatePHI(MemPTy, 2, "vaarg.addr"); + ArgAddr->addIncoming(__saved_reg_area_p, InRegBlock); + ArgAddr->addIncoming(__overflow_area_p, OnStackBlock); + + return Address(ArgAddr, CharUnits::fromQuantity(ArgAlign)); +} + Address HexagonABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty) const { - // FIXME: Someone needs to audit that this handle alignment correctly. - return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*indirect*/ false, - getContext().getTypeInfoInChars(Ty), - CharUnits::fromQuantity(4), - /*AllowHigherAlign*/ true); + + if (getTarget().getTriple().isMusl()) + return EmitVAArgForHexagonLinux(CGF, VAListAddr, Ty); + + return EmitVAArgForHexagon(CGF, VAListAddr, Ty); } //===----------------------------------------------------------------------===// @@ -7676,7 +8589,13 @@ ABIArgInfo LanaiABIInfo::classifyArgumentType(QualType Ty, Ty = EnumTy->getDecl()->getIntegerType(); bool InReg = shouldUseInReg(Ty, State); - if (Ty->isPromotableIntegerType()) { + + // Don't pass >64 bit integers in registers. + if (const auto *EIT = Ty->getAs<ExtIntType>()) + if (EIT->getNumBits() > 64) + return getIndirectResult(Ty, /*ByVal=*/true, State); + + if (isPromotableIntegerTypeForABI(Ty)) { if (InReg) return ABIArgInfo::getDirectInReg(); return ABIArgInfo::getExtend(Ty); @@ -7690,7 +8609,7 @@ namespace { class LanaiTargetCodeGenInfo : public TargetCodeGenInfo { public: LanaiTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT) - : TargetCodeGenInfo(new LanaiABIInfo(CGT)) {} + : TargetCodeGenInfo(std::make_unique<LanaiABIInfo>(CGT)) {} }; } @@ -7730,7 +8649,7 @@ private: EltTys, (STy->getName() + ".coerce").str(), STy->isPacked()); return llvm::StructType::get(getVMContext(), EltTys, STy->isPacked()); } - // Arrary types. + // Array types. if (auto ATy = dyn_cast<llvm::ArrayType>(Ty)) { auto T = ATy->getElementType(); auto NT = coerceKernelArgumentType(T, FromAS, ToAS); @@ -7958,7 +8877,7 @@ ABIArgInfo AMDGPUABIInfo::classifyArgumentType(QualType Ty, class AMDGPUTargetCodeGenInfo : public TargetCodeGenInfo { public: AMDGPUTargetCodeGenInfo(CodeGenTypes &CGT) - : TargetCodeGenInfo(new AMDGPUABIInfo(CGT)) {} + : TargetCodeGenInfo(std::make_unique<AMDGPUABIInfo>(CGT)) {} void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const override; unsigned getOpenCLKernelCallingConv() const override; @@ -7994,23 +8913,13 @@ static bool requiresAMDGPUProtectedVisibility(const Decl *D, (isa<FunctionDecl>(D) && D->hasAttr<CUDAGlobalAttr>()) || (isa<VarDecl>(D) && (D->hasAttr<CUDADeviceAttr>() || D->hasAttr<CUDAConstantAttr>() || - D->hasAttr<HIPPinnedShadowAttr>())); -} - -static bool requiresAMDGPUDefaultVisibility(const Decl *D, - llvm::GlobalValue *GV) { - if (GV->getVisibility() != llvm::GlobalValue::HiddenVisibility) - return false; - - return isa<VarDecl>(D) && D->hasAttr<HIPPinnedShadowAttr>(); + cast<VarDecl>(D)->getType()->isCUDADeviceBuiltinSurfaceType() || + cast<VarDecl>(D)->getType()->isCUDADeviceBuiltinTextureType())); } void AMDGPUTargetCodeGenInfo::setTargetAttributes( const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const { - if (requiresAMDGPUDefaultVisibility(D, GV)) { - GV->setVisibility(llvm::GlobalValue::DefaultVisibility); - GV->setDSOLocal(false); - } else if (requiresAMDGPUProtectedVisibility(D, GV)) { + if (requiresAMDGPUProtectedVisibility(D, GV)) { GV->setVisibility(llvm::GlobalValue::ProtectedVisibility); GV->setDSOLocal(true); } @@ -8035,6 +8944,10 @@ void AMDGPUTargetCodeGenInfo::setTargetAttributes( (M.getTriple().getOS() == llvm::Triple::AMDHSA)) F->addFnAttr("amdgpu-implicitarg-num-bytes", "56"); + if (IsHIPKernel) + F->addFnAttr("uniform-work-group-size", "true"); + + const auto *FlatWGS = FD->getAttr<AMDGPUFlatWorkGroupSizeAttr>(); if (ReqdWGS || FlatWGS) { unsigned Min = 0; @@ -8059,9 +8972,13 @@ void AMDGPUTargetCodeGenInfo::setTargetAttributes( assert(Max == 0 && "Max must be zero"); } else if (IsOpenCLKernel || IsHIPKernel) { // By default, restrict the maximum size to a value specified by - // --gpu-max-threads-per-block=n or its default value. + // --gpu-max-threads-per-block=n or its default value for HIP. + const unsigned OpenCLDefaultMaxWorkGroupSize = 256; + const unsigned DefaultMaxWorkGroupSize = + IsOpenCLKernel ? OpenCLDefaultMaxWorkGroupSize + : M.getLangOpts().GPUMaxThreadsPerBlock; std::string AttrVal = - std::string("1,") + llvm::utostr(M.getLangOpts().GPUMaxThreadsPerBlock); + std::string("1,") + llvm::utostr(DefaultMaxWorkGroupSize); F->addFnAttr("amdgpu-flat-work-group-size", AttrVal); } @@ -8223,7 +9140,7 @@ namespace { class SparcV8TargetCodeGenInfo : public TargetCodeGenInfo { public: SparcV8TargetCodeGenInfo(CodeGenTypes &CGT) - : TargetCodeGenInfo(new SparcV8ABIInfo(CGT)) {} + : TargetCodeGenInfo(std::make_unique<SparcV8ABIInfo>(CGT)) {} }; } // end anonymous namespace @@ -8392,6 +9309,10 @@ SparcV9ABIInfo::classifyType(QualType Ty, unsigned SizeLimit) const { if (Size < 64 && Ty->isIntegerType()) return ABIArgInfo::getExtend(Ty); + if (const auto *EIT = Ty->getAs<ExtIntType>()) + if (EIT->getNumBits() < 64) + return ABIArgInfo::getExtend(Ty); + // Other non-aggregates go in registers. if (!isAggregateTypeForABI(Ty)) return ABIArgInfo::getDirect(); @@ -8485,7 +9406,7 @@ namespace { class SparcV9TargetCodeGenInfo : public TargetCodeGenInfo { public: SparcV9TargetCodeGenInfo(CodeGenTypes &CGT) - : TargetCodeGenInfo(new SparcV9ABIInfo(CGT)) {} + : TargetCodeGenInfo(std::make_unique<SparcV9ABIInfo>(CGT)) {} int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override { return 14; @@ -8578,7 +9499,7 @@ private: class ARCTargetCodeGenInfo : public TargetCodeGenInfo { public: ARCTargetCodeGenInfo(CodeGenTypes &CGT) - : TargetCodeGenInfo(new ARCABIInfo(CGT)) {} + : TargetCodeGenInfo(std::make_unique<ARCABIInfo>(CGT)) {} }; @@ -8641,11 +9562,15 @@ ABIArgInfo ARCABIInfo::classifyArgumentType(QualType Ty, ABIArgInfo::getDirect(Result, 0, nullptr, false); } - return Ty->isPromotableIntegerType() ? - (FreeRegs >= SizeInRegs ? ABIArgInfo::getExtendInReg(Ty) : - ABIArgInfo::getExtend(Ty)) : - (FreeRegs >= SizeInRegs ? ABIArgInfo::getDirectInReg() : - ABIArgInfo::getDirect()); + if (const auto *EIT = Ty->getAs<ExtIntType>()) + if (EIT->getNumBits() > 64) + return getIndirectByValue(Ty); + + return isPromotableIntegerTypeForABI(Ty) + ? (FreeRegs >= SizeInRegs ? ABIArgInfo::getExtendInReg(Ty) + : ABIArgInfo::getExtend(Ty)) + : (FreeRegs >= SizeInRegs ? ABIArgInfo::getDirectInReg() + : ABIArgInfo::getDirect()); } ABIArgInfo ARCABIInfo::classifyReturnType(QualType RetTy) const { @@ -8769,11 +9694,15 @@ public: class XCoreTargetCodeGenInfo : public TargetCodeGenInfo { mutable TypeStringCache TSC; + void emitTargetMD(const Decl *D, llvm::GlobalValue *GV, + const CodeGen::CodeGenModule &M) const; + public: XCoreTargetCodeGenInfo(CodeGenTypes &CGT) - :TargetCodeGenInfo(new XCoreABIInfo(CGT)) {} - void emitTargetMD(const Decl *D, llvm::GlobalValue *GV, - CodeGen::CodeGenModule &M) const override; + : TargetCodeGenInfo(std::make_unique<XCoreABIInfo>(CGT)) {} + void emitTargetMetadata(CodeGen::CodeGenModule &CGM, + const llvm::MapVector<GlobalDecl, StringRef> + &MangledDeclNames) const override; }; } // End anonymous namespace. @@ -8934,11 +9863,13 @@ StringRef TypeStringCache::lookupStr(const IdentifierInfo *ID) { /// The output is tested by test/CodeGen/xcore-stringtype.c. /// static bool getTypeString(SmallStringEnc &Enc, const Decl *D, - CodeGen::CodeGenModule &CGM, TypeStringCache &TSC); + const CodeGen::CodeGenModule &CGM, + TypeStringCache &TSC); /// XCore uses emitTargetMD to emit TypeString metadata for global symbols. -void XCoreTargetCodeGenInfo::emitTargetMD(const Decl *D, llvm::GlobalValue *GV, - CodeGen::CodeGenModule &CGM) const { +void XCoreTargetCodeGenInfo::emitTargetMD( + const Decl *D, llvm::GlobalValue *GV, + const CodeGen::CodeGenModule &CGM) const { SmallStringEnc Enc; if (getTypeString(Enc, D, CGM, TSC)) { llvm::LLVMContext &Ctx = CGM.getModule().getContext(); @@ -8950,6 +9881,21 @@ void XCoreTargetCodeGenInfo::emitTargetMD(const Decl *D, llvm::GlobalValue *GV, } } +void XCoreTargetCodeGenInfo::emitTargetMetadata( + CodeGen::CodeGenModule &CGM, + const llvm::MapVector<GlobalDecl, StringRef> &MangledDeclNames) const { + // Warning, new MangledDeclNames may be appended within this loop. + // We rely on MapVector insertions adding new elements to the end + // of the container. + for (unsigned I = 0; I != MangledDeclNames.size(); ++I) { + auto Val = *(MangledDeclNames.begin() + I); + llvm::GlobalValue *GV = CGM.GetGlobalValue(Val.second); + if (GV) { + const Decl *D = Val.first.getDecl()->getMostRecentDecl(); + emitTargetMD(D, GV, CGM); + } + } +} //===----------------------------------------------------------------------===// // SPIR ABI Implementation //===----------------------------------------------------------------------===// @@ -8958,7 +9904,7 @@ namespace { class SPIRTargetCodeGenInfo : public TargetCodeGenInfo { public: SPIRTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT) - : TargetCodeGenInfo(new DefaultABIInfo(CGT)) {} + : TargetCodeGenInfo(std::make_unique<DefaultABIInfo>(CGT)) {} unsigned getOpenCLKernelCallingConv() const override; }; @@ -9283,7 +10229,8 @@ static bool appendType(SmallStringEnc &Enc, QualType QType, } static bool getTypeString(SmallStringEnc &Enc, const Decl *D, - CodeGen::CodeGenModule &CGM, TypeStringCache &TSC) { + const CodeGen::CodeGenModule &CGM, + TypeStringCache &TSC) { if (!D) return false; @@ -9613,7 +10560,8 @@ ABIArgInfo RISCVABIInfo::classifyArgumentType(QualType Ty, bool IsFixed, uint64_t Size = getContext().getTypeSize(Ty); // Pass floating point values via FPRs if possible. - if (IsFixed && Ty->isFloatingType() && FLen >= Size && ArgFPRsLeft) { + if (IsFixed && Ty->isFloatingType() && !Ty->isComplexType() && + FLen >= Size && ArgFPRsLeft) { ArgFPRsLeft--; return ABIArgInfo::getDirect(); } @@ -9676,6 +10624,15 @@ ABIArgInfo RISCVABIInfo::classifyArgumentType(QualType Ty, bool IsFixed, return extendType(Ty); } + if (const auto *EIT = Ty->getAs<ExtIntType>()) { + if (EIT->getNumBits() < XLen && !MustUseStack) + return extendType(Ty); + if (EIT->getNumBits() > 128 || + (!getContext().getTargetInfo().hasInt128Type() && + EIT->getNumBits() > 64)) + return getNaturalAlignIndirect(Ty, /*ByVal=*/false); + } + return ABIArgInfo::getDirect(); } @@ -9747,7 +10704,7 @@ class RISCVTargetCodeGenInfo : public TargetCodeGenInfo { public: RISCVTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, unsigned XLen, unsigned FLen) - : TargetCodeGenInfo(new RISCVABIInfo(CGT, XLen, FLen)) {} + : TargetCodeGenInfo(std::make_unique<RISCVABIInfo>(CGT, XLen, FLen)) {} void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const override { @@ -9773,6 +10730,56 @@ public: } // namespace //===----------------------------------------------------------------------===// +// VE ABI Implementation. +// +namespace { +class VEABIInfo : public DefaultABIInfo { +public: + VEABIInfo(CodeGenTypes &CGT) : DefaultABIInfo(CGT) {} + +private: + ABIArgInfo classifyReturnType(QualType RetTy) const; + ABIArgInfo classifyArgumentType(QualType RetTy) const; + void computeInfo(CGFunctionInfo &FI) const override; +}; +} // end anonymous namespace + +ABIArgInfo VEABIInfo::classifyReturnType(QualType Ty) const { + if (Ty->isAnyComplexType()) { + return ABIArgInfo::getDirect(); + } + return DefaultABIInfo::classifyReturnType(Ty); +} + +ABIArgInfo VEABIInfo::classifyArgumentType(QualType Ty) const { + if (Ty->isAnyComplexType()) { + return ABIArgInfo::getDirect(); + } + return DefaultABIInfo::classifyArgumentType(Ty); +} + +void VEABIInfo::computeInfo(CGFunctionInfo &FI) const { + + FI.getReturnInfo() = classifyReturnType(FI.getReturnType()); + for (auto &Arg : FI.arguments()) + Arg.info = classifyArgumentType(Arg.type); +} + +namespace { +class VETargetCodeGenInfo : public TargetCodeGenInfo { +public: + VETargetCodeGenInfo(CodeGenTypes &CGT) + : TargetCodeGenInfo(std::make_unique<VEABIInfo>(CGT)) {} + // VE ABI requires the arguments of variadic and prototype-less functions + // are passed in both registers and memory. + bool isNoProtoCallVariadic(const CallArgList &args, + const FunctionNoProtoType *fnType) const override { + return true; + } +}; +} // end anonymous namespace + +//===----------------------------------------------------------------------===// // Driver code //===----------------------------------------------------------------------===// @@ -9824,8 +10831,12 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() { } case llvm::Triple::wasm32: - case llvm::Triple::wasm64: - return SetCGInfo(new WebAssemblyTargetCodeGenInfo(Types)); + case llvm::Triple::wasm64: { + WebAssemblyABIInfo::ABIKind Kind = WebAssemblyABIInfo::MVP; + if (getTarget().getABI() == "experimental-mv") + Kind = WebAssemblyABIInfo::ExperimentalMV; + return SetCGInfo(new WebAssemblyTargetCodeGenInfo(Types, Kind)); + } case llvm::Triple::arm: case llvm::Triple::armeb: @@ -9852,11 +10863,21 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() { return SetCGInfo(new ARMTargetCodeGenInfo(Types, Kind)); } - case llvm::Triple::ppc: + case llvm::Triple::ppc: { + if (Triple.isOSAIX()) + return SetCGInfo(new AIXTargetCodeGenInfo(Types, /*Is64Bit*/ false)); + + bool IsSoftFloat = + CodeGenOpts.FloatABI == "soft" || getTarget().hasFeature("spe"); + bool RetSmallStructInRegABI = + PPC32TargetCodeGenInfo::isStructReturnInRegABI(Triple, CodeGenOpts); return SetCGInfo( - new PPC32TargetCodeGenInfo(Types, CodeGenOpts.FloatABI == "soft" || - getTarget().hasFeature("spe"))); + new PPC32TargetCodeGenInfo(Types, IsSoftFloat, RetSmallStructInRegABI)); + } case llvm::Triple::ppc64: + if (Triple.isOSAIX()) + return SetCGInfo(new AIXTargetCodeGenInfo(Types, /*Is64Bit*/ true)); + if (Triple.isOSBinFormatELF()) { PPC64_SVR4_ABIInfo::ABIKind Kind = PPC64_SVR4_ABIInfo::ELFv1; if (getTarget().getABI() == "elfv2") @@ -9866,8 +10887,8 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() { return SetCGInfo(new PPC64_SVR4_TargetCodeGenInfo(Types, Kind, HasQPX, IsSoftFloat)); - } else - return SetCGInfo(new PPC64TargetCodeGenInfo(Types)); + } + return SetCGInfo(new PPC64TargetCodeGenInfo(Types)); case llvm::Triple::ppc64le: { assert(Triple.isOSBinFormatELF() && "PPC64 LE non-ELF not supported!"); PPC64_SVR4_ABIInfo::ABIKind Kind = PPC64_SVR4_ABIInfo::ELFv2; @@ -9900,8 +10921,9 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() { } case llvm::Triple::systemz: { - bool HasVector = getTarget().getABI() == "vector"; - return SetCGInfo(new SystemZTargetCodeGenInfo(Types, HasVector)); + bool SoftFloat = CodeGenOpts.FloatABI == "soft"; + bool HasVector = !SoftFloat && getTarget().getABI() == "vector"; + return SetCGInfo(new SystemZTargetCodeGenInfo(Types, HasVector, SoftFloat)); } case llvm::Triple::tce: @@ -9959,6 +10981,8 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() { case llvm::Triple::spir: case llvm::Triple::spir64: return SetCGInfo(new SPIRTargetCodeGenInfo(Types)); + case llvm::Triple::ve: + return SetCGInfo(new VETargetCodeGenInfo(Types)); } } @@ -10042,9 +11066,9 @@ llvm::Function *AMDGPUTargetCodeGenInfo::createEnqueuedBlockKernel( auto IP = CGF.Builder.saveIP(); auto *BB = llvm::BasicBlock::Create(C, "entry", F); Builder.SetInsertPoint(BB); - unsigned BlockAlign = CGF.CGM.getDataLayout().getPrefTypeAlignment(BlockTy); + const auto BlockAlign = CGF.CGM.getDataLayout().getPrefTypeAlign(BlockTy); auto *BlockPtr = Builder.CreateAlloca(BlockTy, nullptr); - BlockPtr->setAlignment(llvm::MaybeAlign(BlockAlign)); + BlockPtr->setAlignment(BlockAlign); Builder.CreateAlignedStore(F->arg_begin(), BlockPtr, BlockAlign); auto *Cast = Builder.CreatePointerCast(BlockPtr, InvokeFT->getParamType(0)); llvm::SmallVector<llvm::Value *, 2> Args; |