| //===--- SPIRVCallLowering.cpp - Call lowering ------------------*- C++ -*-===// | |
| // | |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |
| // See https://llvm.org/LICENSE.txt for license information. | |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| // | |
| //===----------------------------------------------------------------------===// | |
| // | |
| // This file implements the lowering of LLVM calls to machine code calls for | |
| // GlobalISel. | |
| // | |
| //===----------------------------------------------------------------------===// | |
| #include "SPIRVCallLowering.h" | |
| #include "MCTargetDesc/SPIRVBaseInfo.h" | |
| #include "SPIRV.h" | |
| #include "SPIRVBuiltins.h" | |
| #include "SPIRVGlobalRegistry.h" | |
| #include "SPIRVISelLowering.h" | |
| #include "SPIRVRegisterInfo.h" | |
| #include "SPIRVSubtarget.h" | |
| #include "SPIRVUtils.h" | |
| #include "llvm/CodeGen/FunctionLoweringInfo.h" | |
| #include "llvm/Support/ModRef.h" | |
| using namespace llvm; | |
| SPIRVCallLowering::SPIRVCallLowering(const SPIRVTargetLowering &TLI, | |
| SPIRVGlobalRegistry *GR) | |
| : CallLowering(&TLI), GR(GR) {} | |
| bool SPIRVCallLowering::lowerReturn(MachineIRBuilder &MIRBuilder, | |
| const Value *Val, ArrayRef<Register> VRegs, | |
| FunctionLoweringInfo &FLI, | |
| Register SwiftErrorVReg) const { | |
| // Currently all return types should use a single register. | |
| // TODO: handle the case of multiple registers. | |
| if (VRegs.size() > 1) | |
| return false; | |
| if (Val) { | |
| const auto &STI = MIRBuilder.getMF().getSubtarget(); | |
| return MIRBuilder.buildInstr(SPIRV::OpReturnValue) | |
| .addUse(VRegs[0]) | |
| .constrainAllUses(MIRBuilder.getTII(), *STI.getRegisterInfo(), | |
| *STI.getRegBankInfo()); | |
| } | |
| MIRBuilder.buildInstr(SPIRV::OpReturn); | |
| return true; | |
| } | |
| // Based on the LLVM function attributes, get a SPIR-V FunctionControl. | |
| static uint32_t getFunctionControl(const Function &F) { | |
| MemoryEffects MemEffects = F.getMemoryEffects(); | |
| uint32_t FuncControl = static_cast<uint32_t>(SPIRV::FunctionControl::None); | |
| if (F.hasFnAttribute(Attribute::AttrKind::NoInline)) | |
| FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::DontInline); | |
| else if (F.hasFnAttribute(Attribute::AttrKind::AlwaysInline)) | |
| FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::Inline); | |
| if (MemEffects.doesNotAccessMemory()) | |
| FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::Pure); | |
| else if (MemEffects.onlyReadsMemory()) | |
| FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::Const); | |
| return FuncControl; | |
| } | |
| static ConstantInt *getConstInt(MDNode *MD, unsigned NumOp) { | |
| if (MD->getNumOperands() > NumOp) { | |
| auto *CMeta = dyn_cast<ConstantAsMetadata>(MD->getOperand(NumOp)); | |
| if (CMeta) | |
| return dyn_cast<ConstantInt>(CMeta->getValue()); | |
| } | |
| return nullptr; | |
| } | |
| // This code restores function args/retvalue types for composite cases | |
| // because the final types should still be aggregate whereas they're i32 | |
| // during the translation to cope with aggregate flattening etc. | |
| static FunctionType *getOriginalFunctionType(const Function &F) { | |
| auto *NamedMD = F.getParent()->getNamedMetadata("spv.cloned_funcs"); | |
| if (NamedMD == nullptr) | |
| return F.getFunctionType(); | |
| Type *RetTy = F.getFunctionType()->getReturnType(); | |
| SmallVector<Type *, 4> ArgTypes; | |
| for (auto &Arg : F.args()) | |
| ArgTypes.push_back(Arg.getType()); | |
| auto ThisFuncMDIt = | |
| std::find_if(NamedMD->op_begin(), NamedMD->op_end(), [&F](MDNode *N) { | |
| return isa<MDString>(N->getOperand(0)) && | |
| cast<MDString>(N->getOperand(0))->getString() == F.getName(); | |
| }); | |
| // TODO: probably one function can have numerous type mutations, | |
| // so we should support this. | |
| if (ThisFuncMDIt != NamedMD->op_end()) { | |
| auto *ThisFuncMD = *ThisFuncMDIt; | |
| MDNode *MD = dyn_cast<MDNode>(ThisFuncMD->getOperand(1)); | |
| assert(MD && "MDNode operand is expected"); | |
| ConstantInt *Const = getConstInt(MD, 0); | |
| if (Const) { | |
| auto *CMeta = dyn_cast<ConstantAsMetadata>(MD->getOperand(1)); | |
| assert(CMeta && "ConstantAsMetadata operand is expected"); | |
| assert(Const->getSExtValue() >= -1); | |
| // Currently -1 indicates return value, greater values mean | |
| // argument numbers. | |
| if (Const->getSExtValue() == -1) | |
| RetTy = CMeta->getType(); | |
| else | |
| ArgTypes[Const->getSExtValue()] = CMeta->getType(); | |
| } | |
| } | |
| return FunctionType::get(RetTy, ArgTypes, F.isVarArg()); | |
| } | |
| static MDString *getKernelArgAttribute(const Function &KernelFunction, | |
| unsigned ArgIdx, | |
| const StringRef AttributeName) { | |
| assert(KernelFunction.getCallingConv() == CallingConv::SPIR_KERNEL && | |
| "Kernel attributes are attached/belong only to kernel functions"); | |
| // Lookup the argument attribute in metadata attached to the kernel function. | |
| MDNode *Node = KernelFunction.getMetadata(AttributeName); | |
| if (Node && ArgIdx < Node->getNumOperands()) | |
| return cast<MDString>(Node->getOperand(ArgIdx)); | |
| // Sometimes metadata containing kernel attributes is not attached to the | |
| // function, but can be found in the named module-level metadata instead. | |
| // For example: | |
| // !opencl.kernels = !{!0} | |
| // !0 = !{void ()* @someKernelFunction, !1, ...} | |
| // !1 = !{!"kernel_arg_addr_space", ...} | |
| // In this case the actual index of searched argument attribute is ArgIdx + 1, | |
| // since the first metadata node operand is occupied by attribute name | |
| // ("kernel_arg_addr_space" in the example above). | |
| unsigned MDArgIdx = ArgIdx + 1; | |
| NamedMDNode *OpenCLKernelsMD = | |
| KernelFunction.getParent()->getNamedMetadata("opencl.kernels"); | |
| if (!OpenCLKernelsMD || OpenCLKernelsMD->getNumOperands() == 0) | |
| return nullptr; | |
| // KernelToMDNodeList contains kernel function declarations followed by | |
| // corresponding MDNodes for each attribute. Search only MDNodes "belonging" | |
| // to the currently lowered kernel function. | |
| MDNode *KernelToMDNodeList = OpenCLKernelsMD->getOperand(0); | |
| bool FoundLoweredKernelFunction = false; | |
| for (const MDOperand &Operand : KernelToMDNodeList->operands()) { | |
| ValueAsMetadata *MaybeValue = dyn_cast<ValueAsMetadata>(Operand); | |
| if (MaybeValue && dyn_cast<Function>(MaybeValue->getValue())->getName() == | |
| KernelFunction.getName()) { | |
| FoundLoweredKernelFunction = true; | |
| continue; | |
| } | |
| if (MaybeValue && FoundLoweredKernelFunction) | |
| return nullptr; | |
| MDNode *MaybeNode = dyn_cast<MDNode>(Operand); | |
| if (FoundLoweredKernelFunction && MaybeNode && | |
| cast<MDString>(MaybeNode->getOperand(0))->getString() == | |
| AttributeName && | |
| MDArgIdx < MaybeNode->getNumOperands()) | |
| return cast<MDString>(MaybeNode->getOperand(MDArgIdx)); | |
| } | |
| return nullptr; | |
| } | |
| static SPIRV::AccessQualifier::AccessQualifier | |
| getArgAccessQual(const Function &F, unsigned ArgIdx) { | |
| if (F.getCallingConv() != CallingConv::SPIR_KERNEL) | |
| return SPIRV::AccessQualifier::ReadWrite; | |
| MDString *ArgAttribute = | |
| getKernelArgAttribute(F, ArgIdx, "kernel_arg_access_qual"); | |
| if (!ArgAttribute) | |
| return SPIRV::AccessQualifier::ReadWrite; | |
| if (ArgAttribute->getString().compare("read_only") == 0) | |
| return SPIRV::AccessQualifier::ReadOnly; | |
| if (ArgAttribute->getString().compare("write_only") == 0) | |
| return SPIRV::AccessQualifier::WriteOnly; | |
| return SPIRV::AccessQualifier::ReadWrite; | |
| } | |
| static std::vector<SPIRV::Decoration::Decoration> | |
| getKernelArgTypeQual(const Function &KernelFunction, unsigned ArgIdx) { | |
| MDString *ArgAttribute = | |
| getKernelArgAttribute(KernelFunction, ArgIdx, "kernel_arg_type_qual"); | |
| if (ArgAttribute && ArgAttribute->getString().compare("volatile") == 0) | |
| return {SPIRV::Decoration::Volatile}; | |
| return {}; | |
| } | |
| static Type *getArgType(const Function &F, unsigned ArgIdx) { | |
| Type *OriginalArgType = getOriginalFunctionType(F)->getParamType(ArgIdx); | |
| if (F.getCallingConv() != CallingConv::SPIR_KERNEL || | |
| isSpecialOpaqueType(OriginalArgType)) | |
| return OriginalArgType; | |
| MDString *MDKernelArgType = | |
| getKernelArgAttribute(F, ArgIdx, "kernel_arg_type"); | |
| if (!MDKernelArgType || !MDKernelArgType->getString().endswith("_t")) | |
| return OriginalArgType; | |
| std::string KernelArgTypeStr = "opencl." + MDKernelArgType->getString().str(); | |
| Type *ExistingOpaqueType = | |
| StructType::getTypeByName(F.getContext(), KernelArgTypeStr); | |
| return ExistingOpaqueType | |
| ? ExistingOpaqueType | |
| : StructType::create(F.getContext(), KernelArgTypeStr); | |
| } | |
| bool SPIRVCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder, | |
| const Function &F, | |
| ArrayRef<ArrayRef<Register>> VRegs, | |
| FunctionLoweringInfo &FLI) const { | |
| assert(GR && "Must initialize the SPIRV type registry before lowering args."); | |
| GR->setCurrentFunc(MIRBuilder.getMF()); | |
| // Assign types and names to all args, and store their types for later. | |
| FunctionType *FTy = getOriginalFunctionType(F); | |
| SmallVector<SPIRVType *, 4> ArgTypeVRegs; | |
| if (VRegs.size() > 0) { | |
| unsigned i = 0; | |
| for (const auto &Arg : F.args()) { | |
| // Currently formal args should use single registers. | |
| // TODO: handle the case of multiple registers. | |
| if (VRegs[i].size() > 1) | |
| return false; | |
| SPIRV::AccessQualifier::AccessQualifier ArgAccessQual = | |
| getArgAccessQual(F, i); | |
| auto *SpirvTy = GR->assignTypeToVReg(getArgType(F, i), VRegs[i][0], | |
| MIRBuilder, ArgAccessQual); | |
| ArgTypeVRegs.push_back(SpirvTy); | |
| if (Arg.hasName()) | |
| buildOpName(VRegs[i][0], Arg.getName(), MIRBuilder); | |
| if (Arg.getType()->isPointerTy()) { | |
| auto DerefBytes = static_cast<unsigned>(Arg.getDereferenceableBytes()); | |
| if (DerefBytes != 0) | |
| buildOpDecorate(VRegs[i][0], MIRBuilder, | |
| SPIRV::Decoration::MaxByteOffset, {DerefBytes}); | |
| } | |
| if (Arg.hasAttribute(Attribute::Alignment)) { | |
| auto Alignment = static_cast<unsigned>( | |
| Arg.getAttribute(Attribute::Alignment).getValueAsInt()); | |
| buildOpDecorate(VRegs[i][0], MIRBuilder, SPIRV::Decoration::Alignment, | |
| {Alignment}); | |
| } | |
| if (Arg.hasAttribute(Attribute::ReadOnly)) { | |
| auto Attr = | |
| static_cast<unsigned>(SPIRV::FunctionParameterAttribute::NoWrite); | |
| buildOpDecorate(VRegs[i][0], MIRBuilder, | |
| SPIRV::Decoration::FuncParamAttr, {Attr}); | |
| } | |
| if (Arg.hasAttribute(Attribute::ZExt)) { | |
| auto Attr = | |
| static_cast<unsigned>(SPIRV::FunctionParameterAttribute::Zext); | |
| buildOpDecorate(VRegs[i][0], MIRBuilder, | |
| SPIRV::Decoration::FuncParamAttr, {Attr}); | |
| } | |
| if (Arg.hasAttribute(Attribute::NoAlias)) { | |
| auto Attr = | |
| static_cast<unsigned>(SPIRV::FunctionParameterAttribute::NoAlias); | |
| buildOpDecorate(VRegs[i][0], MIRBuilder, | |
| SPIRV::Decoration::FuncParamAttr, {Attr}); | |
| } | |
| if (F.getCallingConv() == CallingConv::SPIR_KERNEL) { | |
| std::vector<SPIRV::Decoration::Decoration> ArgTypeQualDecs = | |
| getKernelArgTypeQual(F, i); | |
| for (SPIRV::Decoration::Decoration Decoration : ArgTypeQualDecs) | |
| buildOpDecorate(VRegs[i][0], MIRBuilder, Decoration, {}); | |
| } | |
| MDNode *Node = F.getMetadata("spirv.ParameterDecorations"); | |
| if (Node && i < Node->getNumOperands() && | |
| isa<MDNode>(Node->getOperand(i))) { | |
| MDNode *MD = cast<MDNode>(Node->getOperand(i)); | |
| for (const MDOperand &MDOp : MD->operands()) { | |
| MDNode *MD2 = dyn_cast<MDNode>(MDOp); | |
| assert(MD2 && "Metadata operand is expected"); | |
| ConstantInt *Const = getConstInt(MD2, 0); | |
| assert(Const && "MDOperand should be ConstantInt"); | |
| auto Dec = | |
| static_cast<SPIRV::Decoration::Decoration>(Const->getZExtValue()); | |
| std::vector<uint32_t> DecVec; | |
| for (unsigned j = 1; j < MD2->getNumOperands(); j++) { | |
| ConstantInt *Const = getConstInt(MD2, j); | |
| assert(Const && "MDOperand should be ConstantInt"); | |
| DecVec.push_back(static_cast<uint32_t>(Const->getZExtValue())); | |
| } | |
| buildOpDecorate(VRegs[i][0], MIRBuilder, Dec, DecVec); | |
| } | |
| } | |
| ++i; | |
| } | |
| } | |
| // Generate a SPIR-V type for the function. | |
| auto MRI = MIRBuilder.getMRI(); | |
| Register FuncVReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); | |
| MRI->setRegClass(FuncVReg, &SPIRV::IDRegClass); | |
| if (F.isDeclaration()) | |
| GR->add(&F, &MIRBuilder.getMF(), FuncVReg); | |
| SPIRVType *RetTy = GR->getOrCreateSPIRVType(FTy->getReturnType(), MIRBuilder); | |
| SPIRVType *FuncTy = GR->getOrCreateOpTypeFunctionWithArgs( | |
| FTy, RetTy, ArgTypeVRegs, MIRBuilder); | |
| // Build the OpTypeFunction declaring it. | |
| uint32_t FuncControl = getFunctionControl(F); | |
| MIRBuilder.buildInstr(SPIRV::OpFunction) | |
| .addDef(FuncVReg) | |
| .addUse(GR->getSPIRVTypeID(RetTy)) | |
| .addImm(FuncControl) | |
| .addUse(GR->getSPIRVTypeID(FuncTy)); | |
| // Add OpFunctionParameters. | |
| int i = 0; | |
| for (const auto &Arg : F.args()) { | |
| assert(VRegs[i].size() == 1 && "Formal arg has multiple vregs"); | |
| MRI->setRegClass(VRegs[i][0], &SPIRV::IDRegClass); | |
| MIRBuilder.buildInstr(SPIRV::OpFunctionParameter) | |
| .addDef(VRegs[i][0]) | |
| .addUse(GR->getSPIRVTypeID(ArgTypeVRegs[i])); | |
| if (F.isDeclaration()) | |
| GR->add(&Arg, &MIRBuilder.getMF(), VRegs[i][0]); | |
| i++; | |
| } | |
| // Name the function. | |
| if (F.hasName()) | |
| buildOpName(FuncVReg, F.getName(), MIRBuilder); | |
| // Handle entry points and function linkage. | |
| if (F.getCallingConv() == CallingConv::SPIR_KERNEL) { | |
| auto MIB = MIRBuilder.buildInstr(SPIRV::OpEntryPoint) | |
| .addImm(static_cast<uint32_t>(SPIRV::ExecutionModel::Kernel)) | |
| .addUse(FuncVReg); | |
| addStringImm(F.getName(), MIB); | |
| } else if (F.getLinkage() == GlobalValue::LinkageTypes::ExternalLinkage || | |
| F.getLinkage() == GlobalValue::LinkOnceODRLinkage) { | |
| auto LnkTy = F.isDeclaration() ? SPIRV::LinkageType::Import | |
| : SPIRV::LinkageType::Export; | |
| buildOpDecorate(FuncVReg, MIRBuilder, SPIRV::Decoration::LinkageAttributes, | |
| {static_cast<uint32_t>(LnkTy)}, F.getGlobalIdentifier()); | |
| } | |
| return true; | |
| } | |
| bool SPIRVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder, | |
| CallLoweringInfo &Info) const { | |
| // Currently call returns should have single vregs. | |
| // TODO: handle the case of multiple registers. | |
| if (Info.OrigRet.Regs.size() > 1) | |
| return false; | |
| MachineFunction &MF = MIRBuilder.getMF(); | |
| GR->setCurrentFunc(MF); | |
| FunctionType *FTy = nullptr; | |
| const Function *CF = nullptr; | |
| // Emit a regular OpFunctionCall. If it's an externally declared function, | |
| // be sure to emit its type and function declaration here. It will be hoisted | |
| // globally later. | |
| if (Info.Callee.isGlobal()) { | |
| CF = dyn_cast_or_null<const Function>(Info.Callee.getGlobal()); | |
| // TODO: support constexpr casts and indirect calls. | |
| if (CF == nullptr) | |
| return false; | |
| FTy = getOriginalFunctionType(*CF); | |
| } | |
| Register ResVReg = | |
| Info.OrigRet.Regs.empty() ? Register(0) : Info.OrigRet.Regs[0]; | |
| std::string FuncName = Info.Callee.getGlobal()->getName().str(); | |
| std::string DemangledName = getOclOrSpirvBuiltinDemangledName(FuncName); | |
| const auto *ST = static_cast<const SPIRVSubtarget *>(&MF.getSubtarget()); | |
| // TODO: check that it's OCL builtin, then apply OpenCL_std. | |
| if (!DemangledName.empty() && CF && CF->isDeclaration() && | |
| ST->canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { | |
| const Type *OrigRetTy = Info.OrigRet.Ty; | |
| if (FTy) | |
| OrigRetTy = FTy->getReturnType(); | |
| SmallVector<Register, 8> ArgVRegs; | |
| for (auto Arg : Info.OrigArgs) { | |
| assert(Arg.Regs.size() == 1 && "Call arg has multiple VRegs"); | |
| ArgVRegs.push_back(Arg.Regs[0]); | |
| SPIRVType *SPIRVTy = GR->getOrCreateSPIRVType(Arg.Ty, MIRBuilder); | |
| GR->assignSPIRVTypeToVReg(SPIRVTy, Arg.Regs[0], MIRBuilder.getMF()); | |
| } | |
| if (auto Res = SPIRV::lowerBuiltin( | |
| DemangledName, SPIRV::InstructionSet::OpenCL_std, MIRBuilder, | |
| ResVReg, OrigRetTy, ArgVRegs, GR)) | |
| return *Res; | |
| } | |
| if (CF && CF->isDeclaration() && | |
| !GR->find(CF, &MIRBuilder.getMF()).isValid()) { | |
| // Emit the type info and forward function declaration to the first MBB | |
| // to ensure VReg definition dependencies are valid across all MBBs. | |
| MachineIRBuilder FirstBlockBuilder; | |
| FirstBlockBuilder.setMF(MF); | |
| FirstBlockBuilder.setMBB(*MF.getBlockNumbered(0)); | |
| SmallVector<ArrayRef<Register>, 8> VRegArgs; | |
| SmallVector<SmallVector<Register, 1>, 8> ToInsert; | |
| for (const Argument &Arg : CF->args()) { | |
| if (MIRBuilder.getDataLayout().getTypeStoreSize(Arg.getType()).isZero()) | |
| continue; // Don't handle zero sized types. | |
| ToInsert.push_back( | |
| {MIRBuilder.getMRI()->createGenericVirtualRegister(LLT::scalar(32))}); | |
| VRegArgs.push_back(ToInsert.back()); | |
| } | |
| // TODO: Reuse FunctionLoweringInfo | |
| FunctionLoweringInfo FuncInfo; | |
| lowerFormalArguments(FirstBlockBuilder, *CF, VRegArgs, FuncInfo); | |
| } | |
| // Make sure there's a valid return reg, even for functions returning void. | |
| if (!ResVReg.isValid()) | |
| ResVReg = MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::IDRegClass); | |
| SPIRVType *RetType = | |
| GR->assignTypeToVReg(FTy->getReturnType(), ResVReg, MIRBuilder); | |
| // Emit the OpFunctionCall and its args. | |
| auto MIB = MIRBuilder.buildInstr(SPIRV::OpFunctionCall) | |
| .addDef(ResVReg) | |
| .addUse(GR->getSPIRVTypeID(RetType)) | |
| .add(Info.Callee); | |
| for (const auto &Arg : Info.OrigArgs) { | |
| // Currently call args should have single vregs. | |
| if (Arg.Regs.size() > 1) | |
| return false; | |
| MIB.addUse(Arg.Regs[0]); | |
| } | |
| return MIB.constrainAllUses(MIRBuilder.getTII(), *ST->getRegisterInfo(), | |
| *ST->getRegBankInfo()); | |
| } |