blob: 989bb094cc25ce3b72564018c71c878c6b608767 [file] [log] [blame]
//===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Triple.h"
#include "llvm/ExecutionEngine/JITLink/x86_64.h"
#include "llvm/ExecutionEngine/Orc/OrcABISupport.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/MC/MCDisassembler/MCDisassembler.h"
#include "llvm/MC/MCInstrAnalysis.h"
#include "llvm/Support/Format.h"
#include "llvm/Transforms/Utils/Cloning.h"
#include <sstream>
#define DEBUG_TYPE "orc"
using namespace llvm;
using namespace llvm::orc;
namespace {
class CompileCallbackMaterializationUnit : public orc::MaterializationUnit {
public:
using CompileFunction = JITCompileCallbackManager::CompileFunction;
CompileCallbackMaterializationUnit(SymbolStringPtr Name,
CompileFunction Compile)
: MaterializationUnit(Interface(
SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}), nullptr)),
Name(std::move(Name)), Compile(std::move(Compile)) {}
StringRef getName() const override { return "<Compile Callbacks>"; }
private:
void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
SymbolMap Result;
Result[Name] = JITEvaluatedSymbol(Compile(), JITSymbolFlags::Exported);
// No dependencies, so these calls cannot fail.
cantFail(R->notifyResolved(Result));
cantFail(R->notifyEmitted());
}
void discard(const JITDylib &JD, const SymbolStringPtr &Name) override {
llvm_unreachable("Discard should never occur on a LMU?");
}
SymbolStringPtr Name;
CompileFunction Compile;
};
} // namespace
namespace llvm {
namespace orc {
TrampolinePool::~TrampolinePool() = default;
void IndirectStubsManager::anchor() {}
Expected<JITTargetAddress>
JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) {
if (auto TrampolineAddr = TP->getTrampoline()) {
auto CallbackName =
ES.intern(std::string("cc") + std::to_string(++NextCallbackId));
std::lock_guard<std::mutex> Lock(CCMgrMutex);
AddrToSymbol[*TrampolineAddr] = CallbackName;
cantFail(
CallbacksJD.define(std::make_unique<CompileCallbackMaterializationUnit>(
std::move(CallbackName), std::move(Compile))));
return *TrampolineAddr;
} else
return TrampolineAddr.takeError();
}
JITTargetAddress JITCompileCallbackManager::executeCompileCallback(
JITTargetAddress TrampolineAddr) {
SymbolStringPtr Name;
{
std::unique_lock<std::mutex> Lock(CCMgrMutex);
auto I = AddrToSymbol.find(TrampolineAddr);
// If this address is not associated with a compile callback then report an
// error to the execution session and return ErrorHandlerAddress to the
// callee.
if (I == AddrToSymbol.end()) {
Lock.unlock();
std::string ErrMsg;
{
raw_string_ostream ErrMsgStream(ErrMsg);
ErrMsgStream << "No compile callback for trampoline at "
<< format("0x%016" PRIx64, TrampolineAddr);
}
ES.reportError(
make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()));
return ErrorHandlerAddress;
} else
Name = I->second;
}
if (auto Sym =
ES.lookup(makeJITDylibSearchOrder(
&CallbacksJD, JITDylibLookupFlags::MatchAllSymbols),
Name))
return Sym->getAddress();
else {
llvm::dbgs() << "Didn't find callback.\n";
// If anything goes wrong materializing Sym then report it to the session
// and return the ErrorHandlerAddress;
ES.reportError(Sym.takeError());
return ErrorHandlerAddress;
}
}
Expected<std::unique_ptr<JITCompileCallbackManager>>
createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES,
JITTargetAddress ErrorHandlerAddress) {
switch (T.getArch()) {
default:
return make_error<StringError>(
std::string("No callback manager available for ") + T.str(),
inconvertibleErrorCode());
case Triple::aarch64:
case Triple::aarch64_32: {
typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT;
return CCMgrT::Create(ES, ErrorHandlerAddress);
}
case Triple::x86: {
typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT;
return CCMgrT::Create(ES, ErrorHandlerAddress);
}
case Triple::loongarch64: {
typedef orc::LocalJITCompileCallbackManager<orc::OrcLoongArch64> CCMgrT;
return CCMgrT::Create(ES, ErrorHandlerAddress);
}
case Triple::mips: {
typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Be> CCMgrT;
return CCMgrT::Create(ES, ErrorHandlerAddress);
}
case Triple::mipsel: {
typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Le> CCMgrT;
return CCMgrT::Create(ES, ErrorHandlerAddress);
}
case Triple::mips64:
case Triple::mips64el: {
typedef orc::LocalJITCompileCallbackManager<orc::OrcMips64> CCMgrT;
return CCMgrT::Create(ES, ErrorHandlerAddress);
}
case Triple::riscv64: {
typedef orc::LocalJITCompileCallbackManager<orc::OrcRiscv64> CCMgrT;
return CCMgrT::Create(ES, ErrorHandlerAddress);
}
case Triple::x86_64: {
if (T.getOS() == Triple::OSType::Win32) {
typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT;
return CCMgrT::Create(ES, ErrorHandlerAddress);
} else {
typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT;
return CCMgrT::Create(ES, ErrorHandlerAddress);
}
}
}
}
std::function<std::unique_ptr<IndirectStubsManager>()>
createLocalIndirectStubsManagerBuilder(const Triple &T) {
switch (T.getArch()) {
default:
return [](){
return std::make_unique<
orc::LocalIndirectStubsManager<orc::OrcGenericABI>>();
};
case Triple::aarch64:
case Triple::aarch64_32:
return [](){
return std::make_unique<
orc::LocalIndirectStubsManager<orc::OrcAArch64>>();
};
case Triple::x86:
return [](){
return std::make_unique<
orc::LocalIndirectStubsManager<orc::OrcI386>>();
};
case Triple::loongarch64:
return []() {
return std::make_unique<
orc::LocalIndirectStubsManager<orc::OrcLoongArch64>>();
};
case Triple::mips:
return [](){
return std::make_unique<
orc::LocalIndirectStubsManager<orc::OrcMips32Be>>();
};
case Triple::mipsel:
return [](){
return std::make_unique<
orc::LocalIndirectStubsManager<orc::OrcMips32Le>>();
};
case Triple::mips64:
case Triple::mips64el:
return [](){
return std::make_unique<
orc::LocalIndirectStubsManager<orc::OrcMips64>>();
};
case Triple::riscv64:
return []() {
return std::make_unique<
orc::LocalIndirectStubsManager<orc::OrcRiscv64>>();
};
case Triple::x86_64:
if (T.getOS() == Triple::OSType::Win32) {
return [](){
return std::make_unique<
orc::LocalIndirectStubsManager<orc::OrcX86_64_Win32>>();
};
} else {
return [](){
return std::make_unique<
orc::LocalIndirectStubsManager<orc::OrcX86_64_SysV>>();
};
}
}
}
Constant* createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr) {
Constant *AddrIntVal =
ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr);
Constant *AddrPtrVal =
ConstantExpr::getCast(Instruction::IntToPtr, AddrIntVal,
PointerType::get(&FT, 0));
return AddrPtrVal;
}
GlobalVariable* createImplPointer(PointerType &PT, Module &M,
const Twine &Name, Constant *Initializer) {
auto IP = new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage,
Initializer, Name, nullptr,
GlobalValue::NotThreadLocal, 0, true);
IP->setVisibility(GlobalValue::HiddenVisibility);
return IP;
}
void makeStub(Function &F, Value &ImplPointer) {
assert(F.isDeclaration() && "Can't turn a definition into a stub.");
assert(F.getParent() && "Function isn't in a module.");
Module &M = *F.getParent();
BasicBlock *EntryBlock = BasicBlock::Create(M.getContext(), "entry", &F);
IRBuilder<> Builder(EntryBlock);
LoadInst *ImplAddr = Builder.CreateLoad(F.getType(), &ImplPointer);
std::vector<Value*> CallArgs;
for (auto &A : F.args())
CallArgs.push_back(&A);
CallInst *Call = Builder.CreateCall(F.getFunctionType(), ImplAddr, CallArgs);
Call->setTailCall();
Call->setAttributes(F.getAttributes());
if (F.getReturnType()->isVoidTy())
Builder.CreateRetVoid();
else
Builder.CreateRet(Call);
}
std::vector<GlobalValue *> SymbolLinkagePromoter::operator()(Module &M) {
std::vector<GlobalValue *> PromotedGlobals;
for (auto &GV : M.global_values()) {
bool Promoted = true;
// Rename if necessary.
if (!GV.hasName())
GV.setName("__orc_anon." + Twine(NextId++));
else if (GV.getName().startswith("\01L"))
GV.setName("__" + GV.getName().substr(1) + "." + Twine(NextId++));
else if (GV.hasLocalLinkage())
GV.setName("__orc_lcl." + GV.getName() + "." + Twine(NextId++));
else
Promoted = false;
if (GV.hasLocalLinkage()) {
GV.setLinkage(GlobalValue::ExternalLinkage);
GV.setVisibility(GlobalValue::HiddenVisibility);
Promoted = true;
}
GV.setUnnamedAddr(GlobalValue::UnnamedAddr::None);
if (Promoted)
PromotedGlobals.push_back(&GV);
}
return PromotedGlobals;
}
Function* cloneFunctionDecl(Module &Dst, const Function &F,
ValueToValueMapTy *VMap) {
Function *NewF =
Function::Create(cast<FunctionType>(F.getValueType()),
F.getLinkage(), F.getName(), &Dst);
NewF->copyAttributesFrom(&F);
if (VMap) {
(*VMap)[&F] = NewF;
auto NewArgI = NewF->arg_begin();
for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE;
++ArgI, ++NewArgI)
(*VMap)[&*ArgI] = &*NewArgI;
}
return NewF;
}
void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap,
ValueMaterializer *Materializer,
Function *NewF) {
assert(!OrigF.isDeclaration() && "Nothing to move");
if (!NewF)
NewF = cast<Function>(VMap[&OrigF]);
else
assert(VMap[&OrigF] == NewF && "Incorrect function mapping in VMap.");
assert(NewF && "Function mapping missing from VMap.");
assert(NewF->getParent() != OrigF.getParent() &&
"moveFunctionBody should only be used to move bodies between "
"modules.");
SmallVector<ReturnInst *, 8> Returns; // Ignore returns cloned.
CloneFunctionInto(NewF, &OrigF, VMap,
CloneFunctionChangeType::DifferentModule, Returns, "",
nullptr, nullptr, Materializer);
OrigF.deleteBody();
}
GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
ValueToValueMapTy *VMap) {
GlobalVariable *NewGV = new GlobalVariable(
Dst, GV.getValueType(), GV.isConstant(),
GV.getLinkage(), nullptr, GV.getName(), nullptr,
GV.getThreadLocalMode(), GV.getType()->getAddressSpace());
NewGV->copyAttributesFrom(&GV);
if (VMap)
(*VMap)[&GV] = NewGV;
return NewGV;
}
void moveGlobalVariableInitializer(GlobalVariable &OrigGV,
ValueToValueMapTy &VMap,
ValueMaterializer *Materializer,
GlobalVariable *NewGV) {
assert(OrigGV.hasInitializer() && "Nothing to move");
if (!NewGV)
NewGV = cast<GlobalVariable>(VMap[&OrigGV]);
else
assert(VMap[&OrigGV] == NewGV &&
"Incorrect global variable mapping in VMap.");
assert(NewGV->getParent() != OrigGV.getParent() &&
"moveGlobalVariableInitializer should only be used to move "
"initializers between modules");
NewGV->setInitializer(MapValue(OrigGV.getInitializer(), VMap, RF_None,
nullptr, Materializer));
}
GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA,
ValueToValueMapTy &VMap) {
assert(OrigA.getAliasee() && "Original alias doesn't have an aliasee?");
auto *NewA = GlobalAlias::create(OrigA.getValueType(),
OrigA.getType()->getPointerAddressSpace(),
OrigA.getLinkage(), OrigA.getName(), &Dst);
NewA->copyAttributesFrom(&OrigA);
VMap[&OrigA] = NewA;
return NewA;
}
void cloneModuleFlagsMetadata(Module &Dst, const Module &Src,
ValueToValueMapTy &VMap) {
auto *MFs = Src.getModuleFlagsMetadata();
if (!MFs)
return;
for (auto *MF : MFs->operands())
Dst.addModuleFlag(MapMetadata(MF, VMap));
}
Error addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol &Sym,
jitlink::LinkGraph &G,
MCDisassembler &Disassembler,
MCInstrAnalysis &MIA) {
// AArch64 appears to already come with the necessary relocations. Among other
// architectures, only x86_64 is currently implemented here.
if (G.getTargetTriple().getArch() != Triple::x86_64)
return Error::success();
raw_null_ostream CommentStream;
auto &STI = Disassembler.getSubtargetInfo();
// Determine the function bounds
auto &B = Sym.getBlock();
assert(!B.isZeroFill() && "expected content block");
auto SymAddress = Sym.getAddress();
auto SymStartInBlock =
(const uint8_t *)B.getContent().data() + Sym.getOffset();
auto SymSize = Sym.getSize() ? Sym.getSize() : B.getSize() - Sym.getOffset();
auto Content = ArrayRef(SymStartInBlock, SymSize);
LLVM_DEBUG(dbgs() << "Adding self-relocations to " << Sym.getName() << "\n");
SmallDenseSet<uintptr_t, 8> ExistingRelocations;
for (auto &E : B.edges()) {
if (E.isRelocation())
ExistingRelocations.insert(E.getOffset());
}
size_t I = 0;
while (I < Content.size()) {
MCInst Instr;
uint64_t InstrSize = 0;
uint64_t InstrStart = SymAddress.getValue() + I;
auto DecodeStatus = Disassembler.getInstruction(
Instr, InstrSize, Content.drop_front(I), InstrStart, CommentStream);
if (DecodeStatus != MCDisassembler::Success) {
LLVM_DEBUG(dbgs() << "Aborting due to disassembly failure at address "
<< InstrStart);
return make_error<StringError>(
formatv("failed to disassemble at address {0:x16}", InstrStart),
inconvertibleErrorCode());
}
// Advance to the next instruction.
I += InstrSize;
// Check for a PC-relative address equal to the symbol itself.
auto PCRelAddr =
MIA.evaluateMemoryOperandAddress(Instr, &STI, InstrStart, InstrSize);
if (!PCRelAddr || *PCRelAddr != SymAddress.getValue())
continue;
auto RelocOffInInstr =
MIA.getMemoryOperandRelocationOffset(Instr, InstrSize);
if (!RelocOffInInstr || InstrSize - *RelocOffInInstr != 4) {
LLVM_DEBUG(dbgs() << "Skipping unknown self-relocation at "
<< InstrStart);
continue;
}
auto RelocOffInBlock = orc::ExecutorAddr(InstrStart) + *RelocOffInInstr -
SymAddress + Sym.getOffset();
if (ExistingRelocations.contains(RelocOffInBlock))
continue;
LLVM_DEBUG(dbgs() << "Adding delta32 self-relocation at " << InstrStart);
B.addEdge(jitlink::x86_64::Delta32, RelocOffInBlock, Sym, /*Addend=*/-4);
}
return Error::success();
}
} // End namespace orc.
} // End namespace llvm.