blob: e0af3df9d010f0f87caeb5462fb0993f4d05f273 [file] [log] [blame]
//===- OrcCBindingsStack.h - Orc JIT stack for C bindings -----*- C++ -*---===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H
#define LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H
#include "llvm-c/OrcBindings.h"
#include "llvm-c/TargetMachine.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ExecutionEngine/JITSymbol.h"
#include "llvm/ExecutionEngine/JITEventListener.h"
#include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h"
#include "llvm/ExecutionEngine/Orc/CompileUtils.h"
#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
#include "llvm/ExecutionEngine/Orc/LambdaResolver.h"
#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
#include "llvm/ExecutionEngine/RuntimeDyld.h"
#include "llvm/ExecutionEngine/SectionMemoryManager.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/Mangler.h"
#include "llvm/IR/Module.h"
#include "llvm/Support/CBindingWrapping.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include <algorithm>
#include <cstdint>
#include <functional>
#include <map>
#include <memory>
#include <set>
#include <string>
#include <vector>
namespace llvm {
class OrcCBindingsStack;
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OrcCBindingsStack, LLVMOrcJITStackRef)
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
namespace detail {
// FIXME: Kill this off once the Layer concept becomes an interface.
class GenericLayer {
public:
virtual ~GenericLayer() = default;
virtual JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
bool ExportedSymbolsOnly) = 0;
virtual Error removeModule(orc::VModuleKey K) = 0;
};
template <typename LayerT> class GenericLayerImpl : public GenericLayer {
public:
GenericLayerImpl(LayerT &Layer) : Layer(Layer) {}
JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
bool ExportedSymbolsOnly) override {
return Layer.findSymbolIn(K, Name, ExportedSymbolsOnly);
}
Error removeModule(orc::VModuleKey K) override {
return Layer.removeModule(K);
}
private:
LayerT &Layer;
};
template <>
class GenericLayerImpl<orc::LegacyRTDyldObjectLinkingLayer> : public GenericLayer {
private:
using LayerT = orc::LegacyRTDyldObjectLinkingLayer;
public:
GenericLayerImpl(LayerT &Layer) : Layer(Layer) {}
JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
bool ExportedSymbolsOnly) override {
return Layer.findSymbolIn(K, Name, ExportedSymbolsOnly);
}
Error removeModule(orc::VModuleKey K) override {
return Layer.removeObject(K);
}
private:
LayerT &Layer;
};
template <typename LayerT>
std::unique_ptr<GenericLayerImpl<LayerT>> createGenericLayer(LayerT &Layer) {
return std::make_unique<GenericLayerImpl<LayerT>>(Layer);
}
} // end namespace detail
class OrcCBindingsStack {
public:
using CompileCallbackMgr = orc::JITCompileCallbackManager;
using ObjLayerT = orc::LegacyRTDyldObjectLinkingLayer;
using CompileLayerT = orc::LegacyIRCompileLayer<ObjLayerT, orc::SimpleCompiler>;
using CODLayerT =
orc::LegacyCompileOnDemandLayer<CompileLayerT, CompileCallbackMgr>;
using CallbackManagerBuilder =
std::function<std::unique_ptr<CompileCallbackMgr>()>;
using IndirectStubsManagerBuilder = CODLayerT::IndirectStubsManagerBuilderT;
private:
using OwningObject = object::OwningBinary<object::ObjectFile>;
class CBindingsResolver : public orc::SymbolResolver {
public:
CBindingsResolver(OrcCBindingsStack &Stack,
LLVMOrcSymbolResolverFn ExternalResolver,
void *ExternalResolverCtx)
: Stack(Stack), ExternalResolver(std::move(ExternalResolver)),
ExternalResolverCtx(std::move(ExternalResolverCtx)) {}
orc::SymbolNameSet
getResponsibilitySet(const orc::SymbolNameSet &Symbols) override {
orc::SymbolNameSet Result;
for (auto &S : Symbols) {
if (auto Sym = findSymbol(*S)) {
if (!Sym.getFlags().isStrong())
Result.insert(S);
} else if (auto Err = Sym.takeError()) {
Stack.reportError(std::move(Err));
return orc::SymbolNameSet();
}
}
return Result;
}
orc::SymbolNameSet
lookup(std::shared_ptr<orc::AsynchronousSymbolQuery> Query,
orc::SymbolNameSet Symbols) override {
orc::SymbolNameSet UnresolvedSymbols;
for (auto &S : Symbols) {
if (auto Sym = findSymbol(*S)) {
if (auto Addr = Sym.getAddress()) {
Query->notifySymbolMetRequiredState(
S, JITEvaluatedSymbol(*Addr, Sym.getFlags()));
} else {
Stack.ES.legacyFailQuery(*Query, Addr.takeError());
return orc::SymbolNameSet();
}
} else if (auto Err = Sym.takeError()) {
Stack.ES.legacyFailQuery(*Query, std::move(Err));
return orc::SymbolNameSet();
} else
UnresolvedSymbols.insert(S);
}
if (Query->isComplete())
Query->handleComplete();
return UnresolvedSymbols;
}
private:
JITSymbol findSymbol(const std::string &Name) {
// Search order:
// 1. JIT'd symbols.
// 2. Runtime overrides.
// 3. External resolver (if present).
if (Stack.CODLayer) {
if (auto Sym = Stack.CODLayer->findSymbol(Name, true))
return Sym;
else if (auto Err = Sym.takeError())
return Sym.takeError();
} else {
if (auto Sym = Stack.CompileLayer.findSymbol(Name, true))
return Sym;
else if (auto Err = Sym.takeError())
return Sym.takeError();
}
if (auto Sym = Stack.CXXRuntimeOverrides.searchOverrides(Name))
return Sym;
if (ExternalResolver)
return JITSymbol(ExternalResolver(Name.c_str(), ExternalResolverCtx),
JITSymbolFlags::Exported);
return JITSymbol(nullptr);
}
OrcCBindingsStack &Stack;
LLVMOrcSymbolResolverFn ExternalResolver;
void *ExternalResolverCtx = nullptr;
};
public:
OrcCBindingsStack(TargetMachine &TM,
IndirectStubsManagerBuilder IndirectStubsMgrBuilder)
: CCMgr(createCompileCallbackManager(TM, ES)), DL(TM.createDataLayout()),
IndirectStubsMgr(IndirectStubsMgrBuilder()),
ObjectLayer(
AcknowledgeORCv1Deprecation, ES,
[this](orc::VModuleKey K) {
auto ResolverI = Resolvers.find(K);
assert(ResolverI != Resolvers.end() &&
"No resolver for module K");
auto Resolver = std::move(ResolverI->second);
Resolvers.erase(ResolverI);
return ObjLayerT::Resources{
std::make_shared<SectionMemoryManager>(), Resolver};
},
nullptr,
[this](orc::VModuleKey K, const object::ObjectFile &Obj,
const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) {
this->notifyFinalized(K, Obj, LoadedObjInfo);
},
[this](orc::VModuleKey K, const object::ObjectFile &Obj) {
this->notifyFreed(K, Obj);
}),
CompileLayer(AcknowledgeORCv1Deprecation, ObjectLayer,
orc::SimpleCompiler(TM)),
CODLayer(createCODLayer(ES, CompileLayer, CCMgr.get(),
std::move(IndirectStubsMgrBuilder), Resolvers)),
CXXRuntimeOverrides(
AcknowledgeORCv1Deprecation,
[this](const std::string &S) { return mangle(S); }) {}
Error shutdown() {
// Run any destructors registered with __cxa_atexit.
CXXRuntimeOverrides.runDestructors();
// Run any IR destructors.
for (auto &DtorRunner : IRStaticDestructorRunners)
if (auto Err = DtorRunner.runViaLayer(*this))
return Err;
return Error::success();
}
std::string mangle(StringRef Name) {
std::string MangledName;
{
raw_string_ostream MangledNameStream(MangledName);
Mangler::getNameWithPrefix(MangledNameStream, Name, DL);
}
return MangledName;
}
template <typename PtrTy>
static PtrTy fromTargetAddress(JITTargetAddress Addr) {
return reinterpret_cast<PtrTy>(static_cast<uintptr_t>(Addr));
}
Expected<JITTargetAddress>
createLazyCompileCallback(LLVMOrcLazyCompileCallbackFn Callback,
void *CallbackCtx) {
auto WrappedCallback = [=]() -> JITTargetAddress {
return Callback(wrap(this), CallbackCtx);
};
return CCMgr->getCompileCallback(std::move(WrappedCallback));
}
Error createIndirectStub(StringRef StubName, JITTargetAddress Addr) {
return IndirectStubsMgr->createStub(StubName, Addr,
JITSymbolFlags::Exported);
}
Error setIndirectStubPointer(StringRef Name, JITTargetAddress Addr) {
return IndirectStubsMgr->updatePointer(Name, Addr);
}
template <typename LayerT>
Expected<orc::VModuleKey>
addIRModule(LayerT &Layer, std::unique_ptr<Module> M,
std::unique_ptr<RuntimeDyld::MemoryManager> MemMgr,
LLVMOrcSymbolResolverFn ExternalResolver,
void *ExternalResolverCtx) {
// Attach a data-layout if one isn't already present.
if (M->getDataLayout().isDefault())
M->setDataLayout(DL);
// Record the static constructors and destructors. We have to do this before
// we hand over ownership of the module to the JIT.
std::vector<std::string> CtorNames, DtorNames;
for (auto Ctor : orc::getConstructors(*M))
CtorNames.push_back(mangle(Ctor.Func->getName()));
for (auto Dtor : orc::getDestructors(*M))
DtorNames.push_back(mangle(Dtor.Func->getName()));
// Add the module to the JIT.
auto K = ES.allocateVModule();
Resolvers[K] = std::make_shared<CBindingsResolver>(*this, ExternalResolver,
ExternalResolverCtx);
if (auto Err = Layer.addModule(K, std::move(M)))
return std::move(Err);
KeyLayers[K] = detail::createGenericLayer(Layer);
// Run the static constructors, and save the static destructor runner for
// execution when the JIT is torn down.
orc::LegacyCtorDtorRunner<OrcCBindingsStack> CtorRunner(
AcknowledgeORCv1Deprecation, std::move(CtorNames), K);
if (auto Err = CtorRunner.runViaLayer(*this))
return std::move(Err);
IRStaticDestructorRunners.emplace_back(AcknowledgeORCv1Deprecation,
std::move(DtorNames), K);
return K;
}
Expected<orc::VModuleKey>
addIRModuleEager(std::unique_ptr<Module> M,
LLVMOrcSymbolResolverFn ExternalResolver,
void *ExternalResolverCtx) {
return addIRModule(CompileLayer, std::move(M),
std::make_unique<SectionMemoryManager>(),
std::move(ExternalResolver), ExternalResolverCtx);
}
Expected<orc::VModuleKey>
addIRModuleLazy(std::unique_ptr<Module> M,
LLVMOrcSymbolResolverFn ExternalResolver,
void *ExternalResolverCtx) {
if (!CODLayer)
return make_error<StringError>("Can not add lazy module: No compile "
"callback manager available",
inconvertibleErrorCode());
return addIRModule(*CODLayer, std::move(M),
std::make_unique<SectionMemoryManager>(),
std::move(ExternalResolver), ExternalResolverCtx);
}
Error removeModule(orc::VModuleKey K) {
// FIXME: Should error release the module key?
if (auto Err = KeyLayers[K]->removeModule(K))
return Err;
ES.releaseVModule(K);
KeyLayers.erase(K);
return Error::success();
}
Expected<orc::VModuleKey> addObject(std::unique_ptr<MemoryBuffer> ObjBuffer,
LLVMOrcSymbolResolverFn ExternalResolver,
void *ExternalResolverCtx) {
if (auto Obj = object::ObjectFile::createObjectFile(
ObjBuffer->getMemBufferRef())) {
auto K = ES.allocateVModule();
Resolvers[K] = std::make_shared<CBindingsResolver>(
*this, ExternalResolver, ExternalResolverCtx);
if (auto Err = ObjectLayer.addObject(K, std::move(ObjBuffer)))
return std::move(Err);
KeyLayers[K] = detail::createGenericLayer(ObjectLayer);
return K;
} else
return Obj.takeError();
}
JITSymbol findSymbol(const std::string &Name,
bool ExportedSymbolsOnly) {
if (auto Sym = IndirectStubsMgr->findStub(Name, ExportedSymbolsOnly))
return Sym;
if (CODLayer)
return CODLayer->findSymbol(mangle(Name), ExportedSymbolsOnly);
return CompileLayer.findSymbol(mangle(Name), ExportedSymbolsOnly);
}
JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
bool ExportedSymbolsOnly) {
assert(KeyLayers.count(K) && "looking up symbol in unknown module");
return KeyLayers[K]->findSymbolIn(K, mangle(Name), ExportedSymbolsOnly);
}
Expected<JITTargetAddress> findSymbolAddress(const std::string &Name,
bool ExportedSymbolsOnly) {
if (auto Sym = findSymbol(Name, ExportedSymbolsOnly)) {
// Successful lookup, non-null symbol:
if (auto AddrOrErr = Sym.getAddress())
return *AddrOrErr;
else
return AddrOrErr.takeError();
} else if (auto Err = Sym.takeError()) {
// Lookup failure - report error.
return std::move(Err);
}
// No symbol not found. Return 0.
return 0;
}
Expected<JITTargetAddress> findSymbolAddressIn(orc::VModuleKey K,
const std::string &Name,
bool ExportedSymbolsOnly) {
if (auto Sym = findSymbolIn(K, Name, ExportedSymbolsOnly)) {
// Successful lookup, non-null symbol:
if (auto AddrOrErr = Sym.getAddress())
return *AddrOrErr;
else
return AddrOrErr.takeError();
} else if (auto Err = Sym.takeError()) {
// Lookup failure - report error.
return std::move(Err);
}
// Symbol not found. Return 0.
return 0;
}
const std::string &getErrorMessage() const { return ErrMsg; }
void RegisterJITEventListener(JITEventListener *L) {
if (!L)
return;
EventListeners.push_back(L);
}
void UnregisterJITEventListener(JITEventListener *L) {
if (!L)
return;
auto I = find(reverse(EventListeners), L);
if (I != EventListeners.rend()) {
std::swap(*I, EventListeners.back());
EventListeners.pop_back();
}
}
private:
using ResolverMap =
std::map<orc::VModuleKey, std::shared_ptr<orc::SymbolResolver>>;
static std::unique_ptr<CompileCallbackMgr>
createCompileCallbackManager(TargetMachine &TM, orc::ExecutionSession &ES) {
auto CCMgr = createLocalCompileCallbackManager(TM.getTargetTriple(), ES, 0);
if (!CCMgr) {
// FIXME: It would be good if we could report this somewhere, but we do
// have an instance yet.
logAllUnhandledErrors(CCMgr.takeError(), errs(), "ORC error: ");
return nullptr;
}
return std::move(*CCMgr);
}
static std::unique_ptr<CODLayerT>
createCODLayer(orc::ExecutionSession &ES, CompileLayerT &CompileLayer,
CompileCallbackMgr *CCMgr,
IndirectStubsManagerBuilder IndirectStubsMgrBuilder,
ResolverMap &Resolvers) {
// If there is no compile callback manager available we can not create a
// compile on demand layer.
if (!CCMgr)
return nullptr;
return std::make_unique<CODLayerT>(
AcknowledgeORCv1Deprecation, ES, CompileLayer,
[&Resolvers](orc::VModuleKey K) {
auto ResolverI = Resolvers.find(K);
assert(ResolverI != Resolvers.end() && "No resolver for module K");
return ResolverI->second;
},
[&Resolvers](orc::VModuleKey K,
std::shared_ptr<orc::SymbolResolver> Resolver) {
assert(!Resolvers.count(K) && "Resolver already present");
Resolvers[K] = std::move(Resolver);
},
[](Function &F) { return std::set<Function *>({&F}); }, *CCMgr,
std::move(IndirectStubsMgrBuilder), false);
}
void reportError(Error Err) {
// FIXME: Report errors on the execution session.
logAllUnhandledErrors(std::move(Err), errs(), "ORC error: ");
};
void notifyFinalized(orc::VModuleKey K,
const object::ObjectFile &Obj,
const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) {
uint64_t Key = static_cast<uint64_t>(
reinterpret_cast<uintptr_t>(Obj.getData().data()));
for (auto &Listener : EventListeners)
Listener->notifyObjectLoaded(Key, Obj, LoadedObjInfo);
}
void notifyFreed(orc::VModuleKey K, const object::ObjectFile &Obj) {
uint64_t Key = static_cast<uint64_t>(
reinterpret_cast<uintptr_t>(Obj.getData().data()));
for (auto &Listener : EventListeners)
Listener->notifyFreeingObject(Key);
}
orc::ExecutionSession ES;
std::unique_ptr<CompileCallbackMgr> CCMgr;
std::vector<JITEventListener *> EventListeners;
DataLayout DL;
SectionMemoryManager CCMgrMemMgr;
std::unique_ptr<orc::IndirectStubsManager> IndirectStubsMgr;
ObjLayerT ObjectLayer;
CompileLayerT CompileLayer;
std::unique_ptr<CODLayerT> CODLayer;
std::map<orc::VModuleKey, std::unique_ptr<detail::GenericLayer>> KeyLayers;
orc::LegacyLocalCXXRuntimeOverrides CXXRuntimeOverrides;
std::vector<orc::LegacyCtorDtorRunner<OrcCBindingsStack>> IRStaticDestructorRunners;
std::string ErrMsg;
ResolverMap Resolvers;
};
} // end namespace llvm
#endif // LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H