| // Copyright 2020 The SwiftShader Authors. All Rights Reserved. | |
| // | |
| // Licensed under the Apache License, Version 2.0 (the "License"); | |
| // you may not use this file except in compliance with the License. | |
| // You may obtain a copy of the License at | |
| // | |
| // http://www.apache.org/licenses/LICENSE-2.0 | |
| // | |
| // Unless required by applicable law or agreed to in writing, software | |
| // distributed under the License is distributed on an "AS IS" BASIS, | |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| // See the License for the specific language governing permissions and | |
| // limitations under the License. | |
| #include "LLVMReactor.hpp" | |
| #include "Debug.hpp" | |
| #include "ExecutableMemory.hpp" | |
| #include "Routine.hpp" | |
| // TODO(b/143539525): Eliminate when warning has been fixed. | |
| #ifdef _MSC_VER | |
| __pragma(warning(push)) | |
| __pragma(warning(disable : 4146)) // unary minus operator applied to unsigned type, result still unsigned | |
| #endif | |
| #include "llvm/Analysis/LoopPass.h" | |
| #include "llvm/ExecutionEngine/ExecutionEngine.h" | |
| #include "llvm/ExecutionEngine/JITSymbol.h" | |
| #include "llvm/ExecutionEngine/Orc/CompileUtils.h" | |
| #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" | |
| #include "llvm/ExecutionEngine/Orc/LambdaResolver.h" | |
| #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" | |
| #include "llvm/ExecutionEngine/RTDyldMemoryManager.h" | |
| #include "llvm/ExecutionEngine/SectionMemoryManager.h" | |
| #include "llvm/IR/Constants.h" | |
| #include "llvm/IR/DataLayout.h" | |
| #include "llvm/IR/Function.h" | |
| #include "llvm/IR/GlobalVariable.h" | |
| #include "llvm/IR/LegacyPassManager.h" | |
| #include "llvm/IR/Mangler.h" | |
| #include "llvm/IR/Module.h" | |
| #include "llvm/IR/Verifier.h" | |
| #include "llvm/Support/Compiler.h" | |
| #include "llvm/Support/Error.h" | |
| #include "llvm/Support/Host.h" | |
| #include "llvm/Support/TargetSelect.h" | |
| #include "llvm/Target/TargetOptions.h" | |
| #include "llvm/Transforms/Coroutines.h" | |
| #include "llvm/Transforms/IPO.h" | |
| #include "llvm/Transforms/IPO/PassManagerBuilder.h" | |
| #include "llvm/Transforms/InstCombine/InstCombine.h" | |
| #include "llvm/Transforms/Scalar.h" | |
| #include "llvm/Transforms/Scalar/GVN.h" | |
| #ifdef _MSC_VER | |
| __pragma(warning(pop)) | |
| #endif | |
| #include <atomic> | |
| #include <unordered_map> | |
| #if defined(_WIN64) | |
| extern "C" void __chkstk(); | |
| #elif defined(_WIN32) | |
| extern "C" void _chkstk(); | |
| #endif | |
| #if __has_feature(memory_sanitizer) | |
| # include <sanitizer/msan_interface.h> | |
| #endif | |
| #ifdef __ARM_EABI__ | |
| extern "C" signed __aeabi_idivmod(); | |
| #endif | |
| namespace { | |
| // JITGlobals is a singleton that holds all the immutable machine specific | |
| // information for the host device. | |
| class JITGlobals | |
| { | |
| public: | |
| using TargetMachineSPtr = std::shared_ptr<llvm::TargetMachine>; | |
| static JITGlobals *get(); | |
| const std::string mcpu; | |
| const std::vector<std::string> mattrs; | |
| const char *const march; | |
| const llvm::TargetOptions targetOptions; | |
| const llvm::DataLayout dataLayout; | |
| TargetMachineSPtr createTargetMachine(rr::Optimization::Level optlevel); | |
| private: | |
| static JITGlobals create(); | |
| static llvm::CodeGenOpt::Level toLLVM(rr::Optimization::Level level); | |
| JITGlobals(const char *mcpu, | |
| const std::vector<std::string> &mattrs, | |
| const char *march, | |
| const llvm::TargetOptions &targetOptions, | |
| const llvm::DataLayout &dataLayout); | |
| JITGlobals(const JITGlobals &) = default; | |
| }; | |
| JITGlobals *JITGlobals::get() | |
| { | |
| static JITGlobals instance = create(); | |
| return &instance; | |
| } | |
| JITGlobals::TargetMachineSPtr JITGlobals::createTargetMachine(rr::Optimization::Level optlevel) | |
| { | |
| #ifdef ENABLE_RR_DEBUG_INFO | |
| auto llvmOptLevel = toLLVM(rr::Optimization::Level::None); | |
| #else // ENABLE_RR_DEBUG_INFO | |
| auto llvmOptLevel = toLLVM(optlevel); | |
| #endif // ENABLE_RR_DEBUG_INFO | |
| return TargetMachineSPtr(llvm::EngineBuilder() | |
| .setOptLevel(llvmOptLevel) | |
| .setMCPU(mcpu) | |
| .setMArch(march) | |
| .setMAttrs(mattrs) | |
| .setTargetOptions(targetOptions) | |
| .selectTarget()); | |
| } | |
| JITGlobals JITGlobals::create() | |
| { | |
| struct LLVMInitializer | |
| { | |
| LLVMInitializer() | |
| { | |
| llvm::InitializeNativeTarget(); | |
| llvm::InitializeNativeTargetAsmPrinter(); | |
| llvm::InitializeNativeTargetAsmParser(); | |
| } | |
| }; | |
| static LLVMInitializer initializeLLVM; | |
| auto mcpu = llvm::sys::getHostCPUName(); | |
| llvm::StringMap<bool> features; | |
| bool ok = llvm::sys::getHostCPUFeatures(features); | |
| #if defined(__i386__) || defined(__x86_64__) || \ | |
| (defined(__linux__) && (defined(__arm__) || defined(__aarch64__))) | |
| ASSERT_MSG(ok, "llvm::sys::getHostCPUFeatures returned false"); | |
| #else | |
| (void)ok; // getHostCPUFeatures always returns false on other platforms | |
| #endif | |
| std::vector<std::string> mattrs; | |
| for(auto &feature : features) | |
| { | |
| if(feature.second) { mattrs.push_back(feature.first().str()); } | |
| } | |
| const char *march = nullptr; | |
| #if defined(__x86_64__) | |
| march = "x86-64"; | |
| #elif defined(__i386__) | |
| march = "x86"; | |
| #elif defined(__aarch64__) | |
| march = "arm64"; | |
| #elif defined(__arm__) | |
| march = "arm"; | |
| #elif defined(__mips__) | |
| # if defined(__mips64) | |
| march = "mips64el"; | |
| # else | |
| march = "mipsel"; | |
| # endif | |
| #elif defined(__powerpc64__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ | |
| march = "ppc64le"; | |
| #else | |
| # error "unknown architecture" | |
| #endif | |
| llvm::TargetOptions targetOptions; | |
| targetOptions.UnsafeFPMath = false; | |
| auto targetMachine = std::unique_ptr<llvm::TargetMachine>( | |
| llvm::EngineBuilder() | |
| .setOptLevel(llvm::CodeGenOpt::None) | |
| .setMCPU(mcpu) | |
| .setMArch(march) | |
| .setMAttrs(mattrs) | |
| .setTargetOptions(targetOptions) | |
| .selectTarget()); | |
| auto dataLayout = targetMachine->createDataLayout(); | |
| return JITGlobals(mcpu.data(), mattrs, march, targetOptions, dataLayout); | |
| } | |
| llvm::CodeGenOpt::Level JITGlobals::toLLVM(rr::Optimization::Level level) | |
| { | |
| switch(level) | |
| { | |
| case rr::Optimization::Level::None: return ::llvm::CodeGenOpt::None; | |
| case rr::Optimization::Level::Less: return ::llvm::CodeGenOpt::Less; | |
| case rr::Optimization::Level::Default: return ::llvm::CodeGenOpt::Default; | |
| case rr::Optimization::Level::Aggressive: return ::llvm::CodeGenOpt::Aggressive; | |
| default: UNREACHABLE("Unknown Optimization Level %d", int(level)); | |
| } | |
| return ::llvm::CodeGenOpt::Default; | |
| } | |
| JITGlobals::JITGlobals(const char *mcpu, | |
| const std::vector<std::string> &mattrs, | |
| const char *march, | |
| const llvm::TargetOptions &targetOptions, | |
| const llvm::DataLayout &dataLayout) | |
| : mcpu(mcpu) | |
| , mattrs(mattrs) | |
| , march(march) | |
| , targetOptions(targetOptions) | |
| , dataLayout(dataLayout) | |
| { | |
| } | |
| class MemoryMapper final : public llvm::SectionMemoryManager::MemoryMapper | |
| { | |
| public: | |
| MemoryMapper() {} | |
| ~MemoryMapper() final {} | |
| llvm::sys::MemoryBlock allocateMappedMemory( | |
| llvm::SectionMemoryManager::AllocationPurpose purpose, | |
| size_t numBytes, const llvm::sys::MemoryBlock *const nearBlock, | |
| unsigned flags, std::error_code &errorCode) final | |
| { | |
| errorCode = std::error_code(); | |
| // Round up numBytes to page size. | |
| size_t pageSize = rr::memoryPageSize(); | |
| numBytes = (numBytes + pageSize - 1) & ~(pageSize - 1); | |
| bool need_exec = | |
| purpose == llvm::SectionMemoryManager::AllocationPurpose::Code; | |
| void *addr = rr::allocateMemoryPages( | |
| numBytes, flagsToPermissions(flags), need_exec); | |
| if(!addr) | |
| return llvm::sys::MemoryBlock(); | |
| return llvm::sys::MemoryBlock(addr, numBytes); | |
| } | |
| std::error_code protectMappedMemory(const llvm::sys::MemoryBlock &block, | |
| unsigned flags) | |
| { | |
| // Round down base address to align with a page boundary. This matches | |
| // DefaultMMapper behavior. | |
| void *addr = block.base(); | |
| size_t size = block.allocatedSize(); | |
| size_t pageSize = rr::memoryPageSize(); | |
| addr = reinterpret_cast<void *>( | |
| reinterpret_cast<uintptr_t>(addr) & ~(pageSize - 1)); | |
| size += reinterpret_cast<uintptr_t>(block.base()) - | |
| reinterpret_cast<uintptr_t>(addr); | |
| rr::protectMemoryPages(addr, size, flagsToPermissions(flags)); | |
| return std::error_code(); | |
| } | |
| std::error_code releaseMappedMemory(llvm::sys::MemoryBlock &block) | |
| { | |
| size_t size = block.allocatedSize(); | |
| rr::deallocateMemoryPages(block.base(), size); | |
| return std::error_code(); | |
| } | |
| private: | |
| int flagsToPermissions(unsigned flags) | |
| { | |
| int result = 0; | |
| if(flags & llvm::sys::Memory::MF_READ) | |
| { | |
| result |= rr::PERMISSION_READ; | |
| } | |
| if(flags & llvm::sys::Memory::MF_WRITE) | |
| { | |
| result |= rr::PERMISSION_WRITE; | |
| } | |
| if(flags & llvm::sys::Memory::MF_EXEC) | |
| { | |
| result |= rr::PERMISSION_EXECUTE; | |
| } | |
| return result; | |
| } | |
| }; | |
| template<typename T> | |
| T alignUp(T val, T alignment) | |
| { | |
| return alignment * ((val + alignment - 1) / alignment); | |
| } | |
| void *alignedAlloc(size_t size, size_t alignment) | |
| { | |
| ASSERT(alignment < 256); | |
| auto allocation = new uint8_t[size + sizeof(uint8_t) + alignment]; | |
| auto aligned = allocation; | |
| aligned += sizeof(uint8_t); // Make space for the base-address offset. | |
| aligned = reinterpret_cast<uint8_t *>(alignUp(reinterpret_cast<uintptr_t>(aligned), alignment)); // align | |
| auto offset = static_cast<uint8_t>(aligned - allocation); | |
| aligned[-1] = offset; | |
| return aligned; | |
| } | |
| void alignedFree(void *ptr) | |
| { | |
| auto aligned = reinterpret_cast<uint8_t *>(ptr); | |
| auto offset = aligned[-1]; | |
| auto allocation = aligned - offset; | |
| delete[] allocation; | |
| } | |
| template<typename T> | |
| static void atomicLoad(void *ptr, void *ret, llvm::AtomicOrdering ordering) | |
| { | |
| *reinterpret_cast<T *>(ret) = std::atomic_load_explicit<T>(reinterpret_cast<std::atomic<T> *>(ptr), rr::atomicOrdering(ordering)); | |
| } | |
| template<typename T> | |
| static void atomicStore(void *ptr, void *val, llvm::AtomicOrdering ordering) | |
| { | |
| std::atomic_store_explicit<T>(reinterpret_cast<std::atomic<T> *>(ptr), *reinterpret_cast<T *>(val), rr::atomicOrdering(ordering)); | |
| } | |
| #ifdef __ANDROID__ | |
| template<typename F> | |
| static uint32_t sync_fetch_and_op(uint32_t volatile *ptr, uint32_t val, F f) | |
| { | |
| // Build an arbitrary op out of looped CAS | |
| for(;;) | |
| { | |
| uint32_t expected = *ptr; | |
| uint32_t desired = f(expected, val); | |
| if(expected == __sync_val_compare_and_swap_4(ptr, expected, desired)) | |
| { | |
| return expected; | |
| } | |
| } | |
| } | |
| #endif | |
| void *resolveExternalSymbol(const char *name) | |
| { | |
| struct Atomic | |
| { | |
| static void load(size_t size, void *ptr, void *ret, llvm::AtomicOrdering ordering) | |
| { | |
| switch(size) | |
| { | |
| case 1: atomicLoad<uint8_t>(ptr, ret, ordering); break; | |
| case 2: atomicLoad<uint16_t>(ptr, ret, ordering); break; | |
| case 4: atomicLoad<uint32_t>(ptr, ret, ordering); break; | |
| case 8: atomicLoad<uint64_t>(ptr, ret, ordering); break; | |
| default: | |
| UNIMPLEMENTED_NO_BUG("Atomic::load(size: %d)", int(size)); | |
| } | |
| } | |
| static void store(size_t size, void *ptr, void *ret, llvm::AtomicOrdering ordering) | |
| { | |
| switch(size) | |
| { | |
| case 1: atomicStore<uint8_t>(ptr, ret, ordering); break; | |
| case 2: atomicStore<uint16_t>(ptr, ret, ordering); break; | |
| case 4: atomicStore<uint32_t>(ptr, ret, ordering); break; | |
| case 8: atomicStore<uint64_t>(ptr, ret, ordering); break; | |
| default: | |
| UNIMPLEMENTED_NO_BUG("Atomic::store(size: %d)", int(size)); | |
| } | |
| } | |
| }; | |
| struct F | |
| { | |
| static void nop() {} | |
| static void neverCalled() { UNREACHABLE("Should never be called"); } | |
| static void *coroutine_alloc_frame(size_t size) { return alignedAlloc(size, 16); } | |
| static void coroutine_free_frame(void *ptr) { alignedFree(ptr); } | |
| #ifdef __ANDROID__ | |
| // forwarders since we can't take address of builtins | |
| static void sync_synchronize() { __sync_synchronize(); } | |
| static uint32_t sync_fetch_and_add_4(uint32_t *ptr, uint32_t val) { return __sync_fetch_and_add_4(ptr, val); } | |
| static uint32_t sync_fetch_and_and_4(uint32_t *ptr, uint32_t val) { return __sync_fetch_and_and_4(ptr, val); } | |
| static uint32_t sync_fetch_and_or_4(uint32_t *ptr, uint32_t val) { return __sync_fetch_and_or_4(ptr, val); } | |
| static uint32_t sync_fetch_and_xor_4(uint32_t *ptr, uint32_t val) { return __sync_fetch_and_xor_4(ptr, val); } | |
| static uint32_t sync_fetch_and_sub_4(uint32_t *ptr, uint32_t val) { return __sync_fetch_and_sub_4(ptr, val); } | |
| static uint32_t sync_lock_test_and_set_4(uint32_t *ptr, uint32_t val) { return __sync_lock_test_and_set_4(ptr, val); } | |
| static uint32_t sync_val_compare_and_swap_4(uint32_t *ptr, uint32_t expected, uint32_t desired) { return __sync_val_compare_and_swap_4(ptr, expected, desired); } | |
| static uint32_t sync_fetch_and_max_4(uint32_t *ptr, uint32_t val) | |
| { | |
| return sync_fetch_and_op(ptr, val, [](int32_t a, int32_t b) { return std::max(a, b); }); | |
| } | |
| static uint32_t sync_fetch_and_min_4(uint32_t *ptr, uint32_t val) | |
| { | |
| return sync_fetch_and_op(ptr, val, [](int32_t a, int32_t b) { return std::min(a, b); }); | |
| } | |
| static uint32_t sync_fetch_and_umax_4(uint32_t *ptr, uint32_t val) | |
| { | |
| return sync_fetch_and_op(ptr, val, [](uint32_t a, uint32_t b) { return std::max(a, b); }); | |
| } | |
| static uint32_t sync_fetch_and_umin_4(uint32_t *ptr, uint32_t val) | |
| { | |
| return sync_fetch_and_op(ptr, val, [](uint32_t a, uint32_t b) { return std::min(a, b); }); | |
| } | |
| #endif | |
| }; | |
| class Resolver | |
| { | |
| public: | |
| using FunctionMap = std::unordered_map<std::string, void *>; | |
| FunctionMap functions; | |
| Resolver() | |
| { | |
| #ifdef ENABLE_RR_PRINT | |
| functions.emplace("rr::DebugPrintf", reinterpret_cast<void *>(rr::DebugPrintf)); | |
| #endif | |
| functions.emplace("nop", reinterpret_cast<void *>(F::nop)); | |
| functions.emplace("floorf", reinterpret_cast<void *>(floorf)); | |
| functions.emplace("nearbyintf", reinterpret_cast<void *>(nearbyintf)); | |
| functions.emplace("truncf", reinterpret_cast<void *>(truncf)); | |
| functions.emplace("printf", reinterpret_cast<void *>(printf)); | |
| functions.emplace("puts", reinterpret_cast<void *>(puts)); | |
| functions.emplace("fmodf", reinterpret_cast<void *>(fmodf)); | |
| functions.emplace("sinf", reinterpret_cast<void *>(sinf)); | |
| functions.emplace("cosf", reinterpret_cast<void *>(cosf)); | |
| functions.emplace("asinf", reinterpret_cast<void *>(asinf)); | |
| functions.emplace("acosf", reinterpret_cast<void *>(acosf)); | |
| functions.emplace("atanf", reinterpret_cast<void *>(atanf)); | |
| functions.emplace("sinhf", reinterpret_cast<void *>(sinhf)); | |
| functions.emplace("coshf", reinterpret_cast<void *>(coshf)); | |
| functions.emplace("tanhf", reinterpret_cast<void *>(tanhf)); | |
| functions.emplace("asinhf", reinterpret_cast<void *>(asinhf)); | |
| functions.emplace("acoshf", reinterpret_cast<void *>(acoshf)); | |
| functions.emplace("atanhf", reinterpret_cast<void *>(atanhf)); | |
| functions.emplace("atan2f", reinterpret_cast<void *>(atan2f)); | |
| functions.emplace("powf", reinterpret_cast<void *>(powf)); | |
| functions.emplace("expf", reinterpret_cast<void *>(expf)); | |
| functions.emplace("logf", reinterpret_cast<void *>(logf)); | |
| functions.emplace("exp2f", reinterpret_cast<void *>(exp2f)); | |
| functions.emplace("log2f", reinterpret_cast<void *>(log2f)); | |
| functions.emplace("sin", reinterpret_cast<void *>(static_cast<double (*)(double)>(sin))); | |
| functions.emplace("cos", reinterpret_cast<void *>(static_cast<double (*)(double)>(cos))); | |
| functions.emplace("asin", reinterpret_cast<void *>(static_cast<double (*)(double)>(asin))); | |
| functions.emplace("acos", reinterpret_cast<void *>(static_cast<double (*)(double)>(acos))); | |
| functions.emplace("atan", reinterpret_cast<void *>(static_cast<double (*)(double)>(atan))); | |
| functions.emplace("sinh", reinterpret_cast<void *>(static_cast<double (*)(double)>(sinh))); | |
| functions.emplace("cosh", reinterpret_cast<void *>(static_cast<double (*)(double)>(cosh))); | |
| functions.emplace("tanh", reinterpret_cast<void *>(static_cast<double (*)(double)>(tanh))); | |
| functions.emplace("asinh", reinterpret_cast<void *>(static_cast<double (*)(double)>(asinh))); | |
| functions.emplace("acosh", reinterpret_cast<void *>(static_cast<double (*)(double)>(acosh))); | |
| functions.emplace("atanh", reinterpret_cast<void *>(static_cast<double (*)(double)>(atanh))); | |
| functions.emplace("atan2", reinterpret_cast<void *>(static_cast<double (*)(double, double)>(atan2))); | |
| functions.emplace("pow", reinterpret_cast<void *>(static_cast<double (*)(double, double)>(pow))); | |
| functions.emplace("exp", reinterpret_cast<void *>(static_cast<double (*)(double)>(exp))); | |
| functions.emplace("log", reinterpret_cast<void *>(static_cast<double (*)(double)>(log))); | |
| functions.emplace("exp2", reinterpret_cast<void *>(static_cast<double (*)(double)>(exp2))); | |
| functions.emplace("log2", reinterpret_cast<void *>(static_cast<double (*)(double)>(log2))); | |
| functions.emplace("atomic_load", reinterpret_cast<void *>(Atomic::load)); | |
| functions.emplace("atomic_store", reinterpret_cast<void *>(Atomic::store)); | |
| // FIXME(b/119409619): use an allocator here so we can control all memory allocations | |
| functions.emplace("coroutine_alloc_frame", reinterpret_cast<void *>(F::coroutine_alloc_frame)); | |
| functions.emplace("coroutine_free_frame", reinterpret_cast<void *>(F::coroutine_free_frame)); | |
| #ifdef __APPLE__ | |
| functions.emplace("sincosf_stret", reinterpret_cast<void *>(__sincosf_stret)); | |
| #elif defined(__linux__) | |
| functions.emplace("sincosf", reinterpret_cast<void *>(sincosf)); | |
| #elif defined(_WIN64) | |
| functions.emplace("chkstk", reinterpret_cast<void *>(__chkstk)); | |
| #elif defined(_WIN32) | |
| functions.emplace("chkstk", reinterpret_cast<void *>(_chkstk)); | |
| #endif | |
| #ifdef __ARM_EABI__ | |
| functions.emplace("aeabi_idivmod", reinterpret_cast<void *>(__aeabi_idivmod)); | |
| #endif | |
| #ifdef __ANDROID__ | |
| functions.emplace("aeabi_unwind_cpp_pr0", reinterpret_cast<void *>(F::neverCalled)); | |
| functions.emplace("sync_synchronize", reinterpret_cast<void *>(F::sync_synchronize)); | |
| functions.emplace("sync_fetch_and_add_4", reinterpret_cast<void *>(F::sync_fetch_and_add_4)); | |
| functions.emplace("sync_fetch_and_and_4", reinterpret_cast<void *>(F::sync_fetch_and_and_4)); | |
| functions.emplace("sync_fetch_and_or_4", reinterpret_cast<void *>(F::sync_fetch_and_or_4)); | |
| functions.emplace("sync_fetch_and_xor_4", reinterpret_cast<void *>(F::sync_fetch_and_xor_4)); | |
| functions.emplace("sync_fetch_and_sub_4", reinterpret_cast<void *>(F::sync_fetch_and_sub_4)); | |
| functions.emplace("sync_lock_test_and_set_4", reinterpret_cast<void *>(F::sync_lock_test_and_set_4)); | |
| functions.emplace("sync_val_compare_and_swap_4", reinterpret_cast<void *>(F::sync_val_compare_and_swap_4)); | |
| functions.emplace("sync_fetch_and_max_4", reinterpret_cast<void *>(F::sync_fetch_and_max_4)); | |
| functions.emplace("sync_fetch_and_min_4", reinterpret_cast<void *>(F::sync_fetch_and_min_4)); | |
| functions.emplace("sync_fetch_and_umax_4", reinterpret_cast<void *>(F::sync_fetch_and_umax_4)); | |
| functions.emplace("sync_fetch_and_umin_4", reinterpret_cast<void *>(F::sync_fetch_and_umin_4)); | |
| #endif | |
| #if __has_feature(memory_sanitizer) | |
| functions.emplace("msan_unpoison", reinterpret_cast<void *>(__msan_unpoison)); | |
| #endif | |
| } | |
| }; | |
| static Resolver resolver; | |
| // Trim off any underscores from the start of the symbol. LLVM likes | |
| // to append these on macOS. | |
| const char *trimmed = name; | |
| while(trimmed[0] == '_') { trimmed++; } | |
| auto it = resolver.functions.find(trimmed); | |
| // Missing functions will likely make the module fail in exciting non-obvious ways. | |
| ASSERT_MSG(it != resolver.functions.end(), "Missing external function: '%s'", name); | |
| return it->second; | |
| } | |
| // JITRoutine is a rr::Routine that holds a LLVM JIT session, compiler and | |
| // object layer as each routine may require different target machine | |
| // settings and no Reactor routine directly links against another. | |
| class JITRoutine : public rr::Routine | |
| { | |
| using ObjLayer = llvm::orc::LegacyRTDyldObjectLinkingLayer; | |
| using CompileLayer = llvm::orc::LegacyIRCompileLayer<ObjLayer, llvm::orc::SimpleCompiler>; | |
| public: | |
| #if defined(__clang__) | |
| // TODO(bclayton): Switch to new JIT | |
| // error: 'LegacyIRCompileLayer' is deprecated: ORCv1 layers (layers with the 'Legacy' prefix) are deprecated. | |
| // Please use the ORCv2 IRCompileLayer instead [-Werror,-Wdeprecated-declarations] | |
| # pragma clang diagnostic push | |
| # pragma clang diagnostic ignored "-Wdeprecated-declarations" | |
| #elif defined(__GNUC__) | |
| # pragma GCC diagnostic push | |
| # pragma GCC diagnostic ignored "-Wdeprecated-declarations" | |
| #endif | |
| JITRoutine( | |
| std::unique_ptr<llvm::Module> module, | |
| llvm::Function **funcs, | |
| size_t count, | |
| const rr::Config &config) | |
| : resolver(createLegacyLookupResolver( | |
| session, | |
| [&](const llvm::StringRef &name) { | |
| void *func = resolveExternalSymbol(name.str().c_str()); | |
| if(func != nullptr) | |
| { | |
| return llvm::JITSymbol( | |
| reinterpret_cast<uintptr_t>(func), llvm::JITSymbolFlags::Absolute); | |
| } | |
| return objLayer.findSymbol(name, true); | |
| }, | |
| [](llvm::Error err) { | |
| if(err) | |
| { | |
| // TODO: Log the symbol resolution errors. | |
| return; | |
| } | |
| })) | |
| , targetMachine(JITGlobals::get()->createTargetMachine(config.getOptimization().getLevel())) | |
| , compileLayer(objLayer, llvm::orc::SimpleCompiler(*targetMachine)) | |
| , objLayer( | |
| session, | |
| [this](llvm::orc::VModuleKey) { | |
| return ObjLayer::Resources{ std::make_shared<llvm::SectionMemoryManager>(&memoryMapper), resolver }; | |
| }, | |
| ObjLayer::NotifyLoadedFtor(), | |
| [](llvm::orc::VModuleKey, const llvm::object::ObjectFile &Obj, const llvm::RuntimeDyld::LoadedObjectInfo &L) { | |
| #ifdef ENABLE_RR_DEBUG_INFO | |
| rr::DebugInfo::NotifyObjectEmitted(Obj, L); | |
| #endif // ENABLE_RR_DEBUG_INFO | |
| }, | |
| [](llvm::orc::VModuleKey, const llvm::object::ObjectFile &Obj) { | |
| #ifdef ENABLE_RR_DEBUG_INFO | |
| rr::DebugInfo::NotifyFreeingObject(Obj); | |
| #endif // ENABLE_RR_DEBUG_INFO | |
| }) | |
| , addresses(count) | |
| { | |
| #if defined(__clang__) | |
| # pragma clang diagnostic pop | |
| #elif defined(__GNUC__) | |
| # pragma GCC diagnostic pop | |
| #endif | |
| std::vector<std::string> mangledNames(count); | |
| for(size_t i = 0; i < count; i++) | |
| { | |
| auto func = funcs[i]; | |
| static std::atomic<size_t> numEmittedFunctions = { 0 }; | |
| std::string name = "f" + llvm::Twine(numEmittedFunctions++).str(); | |
| func->setName(name); | |
| func->setLinkage(llvm::GlobalValue::ExternalLinkage); | |
| func->setDoesNotThrow(); | |
| llvm::raw_string_ostream mangledNameStream(mangledNames[i]); | |
| llvm::Mangler::getNameWithPrefix(mangledNameStream, name, JITGlobals::get()->dataLayout); | |
| } | |
| auto moduleKey = session.allocateVModule(); | |
| // Once the module is passed to the compileLayer, the | |
| // llvm::Functions are freed. Make sure funcs are not referenced | |
| // after this point. | |
| funcs = nullptr; | |
| llvm::cantFail(compileLayer.addModule(moduleKey, std::move(module))); | |
| // Resolve the function addresses. | |
| for(size_t i = 0; i < count; i++) | |
| { | |
| auto symbol = compileLayer.findSymbolIn(moduleKey, mangledNames[i], false); | |
| if(auto address = symbol.getAddress()) | |
| { | |
| addresses[i] = reinterpret_cast<void *>(static_cast<intptr_t>(address.get())); | |
| } | |
| } | |
| } | |
| const void *getEntry(int index) const override | |
| { | |
| return addresses[index]; | |
| } | |
| private: | |
| std::shared_ptr<llvm::orc::SymbolResolver> resolver; | |
| std::shared_ptr<llvm::TargetMachine> targetMachine; | |
| llvm::orc::ExecutionSession session; | |
| CompileLayer compileLayer; | |
| MemoryMapper memoryMapper; | |
| ObjLayer objLayer; | |
| std::vector<const void *> addresses; | |
| }; | |
| } // anonymous namespace | |
| namespace rr { | |
| JITBuilder::JITBuilder(const rr::Config &config) | |
| : config(config) | |
| , module(new llvm::Module("", context)) | |
| , builder(new llvm::IRBuilder<>(context)) | |
| { | |
| module->setDataLayout(JITGlobals::get()->dataLayout); | |
| } | |
| void JITBuilder::optimize(const rr::Config &cfg) | |
| { | |
| #ifdef ENABLE_RR_DEBUG_INFO | |
| if(debugInfo != nullptr) | |
| { | |
| return; // Don't optimize if we're generating debug info. | |
| } | |
| #endif // ENABLE_RR_DEBUG_INFO | |
| std::unique_ptr<llvm::legacy::PassManager> passManager( | |
| new llvm::legacy::PassManager()); | |
| for(auto pass : cfg.getOptimization().getPasses()) | |
| { | |
| switch(pass) | |
| { | |
| case rr::Optimization::Pass::Disabled: break; | |
| case rr::Optimization::Pass::CFGSimplification: passManager->add(llvm::createCFGSimplificationPass()); break; | |
| case rr::Optimization::Pass::LICM: passManager->add(llvm::createLICMPass()); break; | |
| case rr::Optimization::Pass::AggressiveDCE: passManager->add(llvm::createAggressiveDCEPass()); break; | |
| case rr::Optimization::Pass::GVN: passManager->add(llvm::createGVNPass()); break; | |
| case rr::Optimization::Pass::InstructionCombining: passManager->add(llvm::createInstructionCombiningPass()); break; | |
| case rr::Optimization::Pass::Reassociate: passManager->add(llvm::createReassociatePass()); break; | |
| case rr::Optimization::Pass::DeadStoreElimination: passManager->add(llvm::createDeadStoreEliminationPass()); break; | |
| case rr::Optimization::Pass::SCCP: passManager->add(llvm::createSCCPPass()); break; | |
| case rr::Optimization::Pass::ScalarReplAggregates: passManager->add(llvm::createSROAPass()); break; | |
| case rr::Optimization::Pass::EarlyCSEPass: passManager->add(llvm::createEarlyCSEPass()); break; | |
| default: | |
| UNREACHABLE("pass: %d", int(pass)); | |
| } | |
| } | |
| passManager->run(*module); | |
| } | |
| std::shared_ptr<rr::Routine> JITBuilder::acquireRoutine(llvm::Function **funcs, size_t count, const rr::Config &cfg) | |
| { | |
| ASSERT(module); | |
| return std::make_shared<JITRoutine>(std::move(module), funcs, count, cfg); | |
| } | |
| } // namespace rr |