blob: 0a4e9d2cea87af6d209cdd8023d536f9e213865f [file] [log] [blame]
// Copyright 2016 The SwiftShader Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "Debug.hpp"
#include "Print.hpp"
#include "Reactor.hpp"
#include "ReactorDebugInfo.hpp"
#include "SIMD.hpp"
#include "ExecutableMemory.hpp"
#include "Optimizer.hpp"
#include "PragmaInternals.hpp"
#include "src/IceCfg.h"
#include "src/IceCfgNode.h"
#include "src/IceELFObjectWriter.h"
#include "src/IceELFStreamer.h"
#include "src/IceGlobalContext.h"
#include "src/IceGlobalInits.h"
#include "src/IceTypes.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/ManagedStatic.h"
#include "llvm/Support/raw_os_ostream.h"
#include "marl/event.h"
#if __has_feature(memory_sanitizer)
# include <sanitizer/msan_interface.h>
#endif
#if defined(_WIN32)
# ifndef WIN32_LEAN_AND_MEAN
# define WIN32_LEAN_AND_MEAN
# endif // !WIN32_LEAN_AND_MEAN
# ifndef NOMINMAX
# define NOMINMAX
# endif // !NOMINMAX
# include <Windows.h>
#endif
#include <array>
#include <cmath>
#include <iostream>
#include <limits>
#include <mutex>
// Subzero utility functions
// These functions only accept and return Subzero (Ice) types, and do not access any globals.
namespace {
namespace sz {
Ice::Cfg *createFunction(Ice::GlobalContext *context, Ice::Type returnType, const std::vector<Ice::Type> &paramTypes)
{
uint32_t sequenceNumber = 0;
auto *function = Ice::Cfg::create(context, sequenceNumber).release();
function->setStackSizeLimit(512 * 1024); // 512 KiB
Ice::CfgLocalAllocatorScope allocScope{ function };
for(auto type : paramTypes)
{
Ice::Variable *arg = function->makeVariable(type);
function->addArg(arg);
}
Ice::CfgNode *node = function->makeNode();
function->setEntryNode(node);
return function;
}
Ice::Type getPointerType(Ice::Type elementType)
{
if(sizeof(void *) == 8)
{
return Ice::IceType_i64;
}
else
{
return Ice::IceType_i32;
}
}
Ice::Variable *allocateStackVariable(Ice::Cfg *function, Ice::Type type, int arraySize = 0)
{
int typeSize = Ice::typeWidthInBytes(type);
int totalSize = typeSize * (arraySize ? arraySize : 1);
auto bytes = Ice::ConstantInteger32::create(function->getContext(), Ice::IceType_i32, totalSize);
auto address = function->makeVariable(getPointerType(type));
auto alloca = Ice::InstAlloca::create(function, address, bytes, typeSize); // SRoA depends on the alignment to match the type size.
function->getEntryNode()->getInsts().push_front(alloca);
ASSERT(!rr::getPragmaState(rr::InitializeLocalVariables) && "Subzero does not support initializing local variables");
return address;
}
Ice::Constant *getConstantPointer(Ice::GlobalContext *context, const void *ptr)
{
if(sizeof(void *) == 8)
{
return context->getConstantInt64(reinterpret_cast<intptr_t>(ptr));
}
else
{
return context->getConstantInt32(reinterpret_cast<intptr_t>(ptr));
}
}
// TODO(amaiorano): remove this prototype once these are moved to separate header/cpp
Ice::Variable *createTruncate(Ice::Cfg *function, Ice::CfgNode *basicBlock, Ice::Operand *from, Ice::Type toType);
// Wrapper for calls on C functions with Ice types
Ice::Variable *Call(Ice::Cfg *function, Ice::CfgNode *basicBlock, Ice::Type retTy, Ice::Operand *callTarget, const std::vector<Ice::Operand *> &iceArgs, bool isVariadic)
{
Ice::Variable *ret = nullptr;
// Subzero doesn't support boolean return values. Replace with an i32 temporarily,
// then truncate result to bool.
// TODO(b/151158858): Add support to Subzero's InstCall for bool-returning functions
const bool returningBool = (retTy == Ice::IceType_i1);
if(returningBool)
{
ret = function->makeVariable(Ice::IceType_i32);
}
else if(retTy != Ice::IceType_void)
{
ret = function->makeVariable(retTy);
}
auto call = Ice::InstCall::create(function, iceArgs.size(), ret, callTarget, false, false, isVariadic);
for(auto arg : iceArgs)
{
call->addArg(arg);
}
basicBlock->appendInst(call);
if(returningBool)
{
// Truncate result to bool so that if any (lsb) bits were set, result will be true
ret = createTruncate(function, basicBlock, ret, Ice::IceType_i1);
}
return ret;
}
Ice::Variable *Call(Ice::Cfg *function, Ice::CfgNode *basicBlock, Ice::Type retTy, const void *fptr, const std::vector<Ice::Operand *> &iceArgs, bool isVariadic)
{
Ice::Operand *callTarget = getConstantPointer(function->getContext(), fptr);
return Call(function, basicBlock, retTy, callTarget, iceArgs, isVariadic);
}
// Wrapper for calls on C functions with Ice types
template<typename Return, typename... CArgs, typename... RArgs>
Ice::Variable *Call(Ice::Cfg *function, Ice::CfgNode *basicBlock, Return(fptr)(CArgs...), RArgs &&...args)
{
static_assert(sizeof...(CArgs) == sizeof...(RArgs), "Expected number of args don't match");
Ice::Type retTy = T(rr::CToReactorT<Return>::type());
std::vector<Ice::Operand *> iceArgs{ std::forward<RArgs>(args)... };
return Call(function, basicBlock, retTy, reinterpret_cast<const void *>(fptr), iceArgs, false);
}
Ice::Variable *createTruncate(Ice::Cfg *function, Ice::CfgNode *basicBlock, Ice::Operand *from, Ice::Type toType)
{
Ice::Variable *to = function->makeVariable(toType);
Ice::InstCast *cast = Ice::InstCast::create(function, Ice::InstCast::Trunc, to, from);
basicBlock->appendInst(cast);
return to;
}
Ice::Variable *createLoad(Ice::Cfg *function, Ice::CfgNode *basicBlock, Ice::Operand *ptr, Ice::Type type, unsigned int align)
{
Ice::Variable *result = function->makeVariable(type);
auto load = Ice::InstLoad::create(function, result, ptr, align);
basicBlock->appendInst(load);
return result;
}
} // namespace sz
} // namespace
namespace rr {
class ELFMemoryStreamer;
class CoroutineGenerator;
} // namespace rr
namespace {
// Used to automatically invoke llvm_shutdown() when driver is unloaded
llvm::llvm_shutdown_obj llvmShutdownObj;
Ice::GlobalContext *context = nullptr;
Ice::Cfg *function = nullptr;
Ice::CfgNode *entryBlock = nullptr;
Ice::CfgNode *basicBlockTop = nullptr;
Ice::CfgNode *basicBlock = nullptr;
Ice::CfgLocalAllocatorScope *allocator = nullptr;
rr::ELFMemoryStreamer *routine = nullptr;
std::mutex codegenMutex;
Ice::ELFFileStreamer *elfFile = nullptr;
Ice::Fdstream *out = nullptr;
// Coroutine globals
rr::Type *coroYieldType = nullptr;
std::shared_ptr<rr::CoroutineGenerator> coroGen;
marl::Scheduler &getOrCreateScheduler()
{
static auto scheduler = [] {
marl::Scheduler::Config cfg;
cfg.setWorkerThreadCount(8);
return std::make_unique<marl::Scheduler>(cfg);
}();
return *scheduler;
}
rr::Nucleus::OptimizerCallback *optimizerCallback = nullptr;
} // Anonymous namespace
namespace {
#if !defined(__i386__) && defined(_M_IX86)
# define __i386__ 1
#endif
#if !defined(__x86_64__) && (defined(_M_AMD64) || defined(_M_X64))
# define __x86_64__ 1
#endif
Ice::OptLevel toIce(int level)
{
switch(level)
{
// Note that O0 and O1 are not implemented by Subzero
case 0: return Ice::Opt_m1;
case 1: return Ice::Opt_m1;
case 2: return Ice::Opt_2;
case 3: return Ice::Opt_2;
default: UNREACHABLE("Unknown Optimization Level %d", int(level));
}
return Ice::Opt_2;
}
Ice::Intrinsics::MemoryOrder stdToIceMemoryOrder(std::memory_order memoryOrder)
{
switch(memoryOrder)
{
case std::memory_order_relaxed: return Ice::Intrinsics::MemoryOrderRelaxed;
case std::memory_order_consume: return Ice::Intrinsics::MemoryOrderConsume;
case std::memory_order_acquire: return Ice::Intrinsics::MemoryOrderAcquire;
case std::memory_order_release: return Ice::Intrinsics::MemoryOrderRelease;
case std::memory_order_acq_rel: return Ice::Intrinsics::MemoryOrderAcquireRelease;
case std::memory_order_seq_cst: return Ice::Intrinsics::MemoryOrderSequentiallyConsistent;
}
return Ice::Intrinsics::MemoryOrderInvalid;
}
class CPUID
{
public:
const static bool ARM;
const static bool SSE4_1;
private:
static void cpuid(int registers[4], int info)
{
#if defined(__i386__) || defined(__x86_64__)
# if defined(_WIN32)
__cpuid(registers, info);
# else
__asm volatile("cpuid"
: "=a"(registers[0]), "=b"(registers[1]), "=c"(registers[2]), "=d"(registers[3])
: "a"(info));
# endif
#else
registers[0] = 0;
registers[1] = 0;
registers[2] = 0;
registers[3] = 0;
#endif
}
constexpr static bool detectARM()
{
#if defined(__arm__) || defined(__aarch64__)
return true;
#elif defined(__i386__) || defined(__x86_64__)
return false;
#elif defined(__mips__)
return false;
#else
# error "Unknown architecture"
#endif
}
static bool detectSSE4_1()
{
#if defined(__i386__) || defined(__x86_64__)
int registers[4];
cpuid(registers, 1);
return (registers[2] & 0x00080000) != 0;
#else
return false;
#endif
}
};
constexpr bool CPUID::ARM = CPUID::detectARM();
const bool CPUID::SSE4_1 = CPUID::detectSSE4_1();
constexpr bool emulateIntrinsics = false;
constexpr bool emulateMismatchedBitCast = CPUID::ARM;
constexpr bool subzeroDumpEnabled = false;
constexpr bool subzeroEmitTextAsm = false;
#if !ALLOW_DUMP
static_assert(!subzeroDumpEnabled, "Compile Subzero with ALLOW_DUMP=1 for subzeroDumpEnabled");
static_assert(!subzeroEmitTextAsm, "Compile Subzero with ALLOW_DUMP=1 for subzeroEmitTextAsm");
#endif
} // anonymous namespace
namespace rr {
const int SIMD::Width = 4;
std::string Caps::backendName()
{
return "Subzero";
}
bool Caps::coroutinesSupported()
{
return true;
}
bool Caps::fmaIsFast()
{
// TODO(b/214591655): Subzero currently never emits FMA instructions. std::fma() is called instead.
return false;
}
enum EmulatedType
{
EmulatedShift = 16,
EmulatedV2 = 2 << EmulatedShift,
EmulatedV4 = 4 << EmulatedShift,
EmulatedV8 = 8 << EmulatedShift,
EmulatedBits = EmulatedV2 | EmulatedV4 | EmulatedV8,
Type_v2i32 = Ice::IceType_v4i32 | EmulatedV2,
Type_v4i16 = Ice::IceType_v8i16 | EmulatedV4,
Type_v2i16 = Ice::IceType_v8i16 | EmulatedV2,
Type_v8i8 = Ice::IceType_v16i8 | EmulatedV8,
Type_v4i8 = Ice::IceType_v16i8 | EmulatedV4,
Type_v2f32 = Ice::IceType_v4f32 | EmulatedV2,
};
class Value : public Ice::Operand
{};
class SwitchCases : public Ice::InstSwitch
{};
class BasicBlock : public Ice::CfgNode
{};
Ice::Type T(Type *t)
{
static_assert(static_cast<unsigned int>(Ice::IceType_NUM) < static_cast<unsigned int>(EmulatedBits), "Ice::Type overlaps with our emulated types!");
return (Ice::Type)(reinterpret_cast<std::intptr_t>(t) & ~EmulatedBits);
}
Type *T(Ice::Type t)
{
return reinterpret_cast<Type *>(t);
}
Type *T(EmulatedType t)
{
return reinterpret_cast<Type *>(t);
}
std::vector<Ice::Type> T(const std::vector<Type *> &types)
{
std::vector<Ice::Type> result;
result.reserve(types.size());
for(auto &t : types)
{
result.push_back(T(t));
}
return result;
}
Value *V(Ice::Operand *v)
{
return reinterpret_cast<Value *>(v);
}
Ice::Operand *V(Value *v)
{
return reinterpret_cast<Ice::Operand *>(v);
}
std::vector<Ice::Operand *> V(const std::vector<Value *> &values)
{
std::vector<Ice::Operand *> result;
result.reserve(values.size());
for(auto &v : values)
{
result.push_back(V(v));
}
return result;
}
BasicBlock *B(Ice::CfgNode *b)
{
return reinterpret_cast<BasicBlock *>(b);
}
static size_t typeSize(Type *type)
{
if(reinterpret_cast<std::intptr_t>(type) & EmulatedBits)
{
switch(reinterpret_cast<std::intptr_t>(type))
{
case Type_v2i32: return 8;
case Type_v4i16: return 8;
case Type_v2i16: return 4;
case Type_v8i8: return 8;
case Type_v4i8: return 4;
case Type_v2f32: return 8;
default: ASSERT(false);
}
}
return Ice::typeWidthInBytes(T(type));
}
static void finalizeFunction()
{
// Create a return if none was added
if(::basicBlock->getInsts().empty() || ::basicBlock->getInsts().back().getKind() != Ice::Inst::Ret)
{
Nucleus::createRetVoid();
}
// Connect the entry block to the top of the initial basic block
auto br = Ice::InstBr::create(::function, ::basicBlockTop);
::entryBlock->appendInst(br);
}
using ElfHeader = std::conditional<sizeof(void *) == 8, Elf64_Ehdr, Elf32_Ehdr>::type;
using SectionHeader = std::conditional<sizeof(void *) == 8, Elf64_Shdr, Elf32_Shdr>::type;
inline const SectionHeader *sectionHeader(const ElfHeader *elfHeader)
{
return reinterpret_cast<const SectionHeader *>((intptr_t)elfHeader + elfHeader->e_shoff);
}
inline const SectionHeader *elfSection(const ElfHeader *elfHeader, int index)
{
return &sectionHeader(elfHeader)[index];
}
static void *relocateSymbol(const ElfHeader *elfHeader, const Elf32_Rel &relocation, const SectionHeader &relocationTable)
{
const SectionHeader *target = elfSection(elfHeader, relocationTable.sh_info);
uint32_t index = relocation.getSymbol();
int table = relocationTable.sh_link;
void *symbolValue = nullptr;
if(index != SHN_UNDEF)
{
if(table == SHN_UNDEF) return nullptr;
const SectionHeader *symbolTable = elfSection(elfHeader, table);
uint32_t symtab_entries = symbolTable->sh_size / symbolTable->sh_entsize;
if(index >= symtab_entries)
{
ASSERT(index < symtab_entries && "Symbol Index out of range");
return nullptr;
}
intptr_t symbolAddress = (intptr_t)elfHeader + symbolTable->sh_offset;
Elf32_Sym &symbol = ((Elf32_Sym *)symbolAddress)[index];
uint16_t section = symbol.st_shndx;
if(section != SHN_UNDEF && section < SHN_LORESERVE)
{
const SectionHeader *target = elfSection(elfHeader, symbol.st_shndx);
symbolValue = reinterpret_cast<void *>((intptr_t)elfHeader + symbol.st_value + target->sh_offset);
}
else
{
return nullptr;
}
}
intptr_t address = (intptr_t)elfHeader + target->sh_offset;
unaligned_ptr<int32_t> patchSite = (int32_t *)(address + relocation.r_offset);
if(CPUID::ARM)
{
switch(relocation.getType())
{
case R_ARM_NONE:
// No relocation
break;
case R_ARM_MOVW_ABS_NC:
{
uint32_t thumb = 0; // Calls to Thumb code not supported.
uint32_t lo = (uint32_t)(intptr_t)symbolValue | thumb;
*patchSite = (*patchSite & 0xFFF0F000) | ((lo & 0xF000) << 4) | (lo & 0x0FFF);
}
break;
case R_ARM_MOVT_ABS:
{
uint32_t hi = (uint32_t)(intptr_t)(symbolValue) >> 16;
*patchSite = (*patchSite & 0xFFF0F000) | ((hi & 0xF000) << 4) | (hi & 0x0FFF);
}
break;
default:
ASSERT(false && "Unsupported relocation type");
return nullptr;
}
}
else
{
switch(relocation.getType())
{
case R_386_NONE:
// No relocation
break;
case R_386_32:
*patchSite = (int32_t)((intptr_t)symbolValue + *patchSite);
break;
case R_386_PC32:
*patchSite = (int32_t)((intptr_t)symbolValue + *patchSite - (intptr_t)patchSite);
break;
default:
ASSERT(false && "Unsupported relocation type");
return nullptr;
}
}
return symbolValue;
}
static void *relocateSymbol(const ElfHeader *elfHeader, const Elf64_Rela &relocation, const SectionHeader &relocationTable)
{
const SectionHeader *target = elfSection(elfHeader, relocationTable.sh_info);
uint32_t index = relocation.getSymbol();
int table = relocationTable.sh_link;
void *symbolValue = nullptr;
if(index != SHN_UNDEF)
{
if(table == SHN_UNDEF) return nullptr;
const SectionHeader *symbolTable = elfSection(elfHeader, table);
uint32_t symtab_entries = symbolTable->sh_size / symbolTable->sh_entsize;
if(index >= symtab_entries)
{
ASSERT(index < symtab_entries && "Symbol Index out of range");
return nullptr;
}
intptr_t symbolAddress = (intptr_t)elfHeader + symbolTable->sh_offset;
Elf64_Sym &symbol = ((Elf64_Sym *)symbolAddress)[index];
uint16_t section = symbol.st_shndx;
if(section != SHN_UNDEF && section < SHN_LORESERVE)
{
const SectionHeader *target = elfSection(elfHeader, symbol.st_shndx);
symbolValue = reinterpret_cast<void *>((intptr_t)elfHeader + symbol.st_value + target->sh_offset);
}
else
{
return nullptr;
}
}
intptr_t address = (intptr_t)elfHeader + target->sh_offset;
unaligned_ptr<int32_t> patchSite32 = (int32_t *)(address + relocation.r_offset);
unaligned_ptr<int64_t> patchSite64 = (int64_t *)(address + relocation.r_offset);
switch(relocation.getType())
{
case R_X86_64_NONE:
// No relocation
break;
case R_X86_64_64:
*patchSite64 = (int64_t)((intptr_t)symbolValue + *patchSite64 + relocation.r_addend);
break;
case R_X86_64_PC32:
*patchSite32 = (int32_t)((intptr_t)symbolValue + *patchSite32 - (intptr_t)patchSite32 + relocation.r_addend);
break;
case R_X86_64_32S:
*patchSite32 = (int32_t)((intptr_t)symbolValue + *patchSite32 + relocation.r_addend);
break;
default:
ASSERT(false && "Unsupported relocation type");
return nullptr;
}
return symbolValue;
}
struct EntryPoint
{
const void *entry;
size_t codeSize = 0;
};
std::vector<EntryPoint> loadImage(uint8_t *const elfImage, const std::vector<const char *> &functionNames)
{
ASSERT(functionNames.size() > 0);
std::vector<EntryPoint> entryPoints(functionNames.size());
ElfHeader *elfHeader = (ElfHeader *)elfImage;
// TODO: assert?
if(!elfHeader->checkMagic())
{
return {};
}
// Expect ELF bitness to match platform
ASSERT(sizeof(void *) == 8 ? elfHeader->getFileClass() == ELFCLASS64 : elfHeader->getFileClass() == ELFCLASS32);
#if defined(__i386__)
ASSERT(sizeof(void *) == 4 && elfHeader->e_machine == EM_386);
#elif defined(__x86_64__)
ASSERT(sizeof(void *) == 8 && elfHeader->e_machine == EM_X86_64);
#elif defined(__arm__)
ASSERT(sizeof(void *) == 4 && elfHeader->e_machine == EM_ARM);
#elif defined(__aarch64__)
ASSERT(sizeof(void *) == 8 && elfHeader->e_machine == EM_AARCH64);
#elif defined(__mips__)
ASSERT(sizeof(void *) == 4 && elfHeader->e_machine == EM_MIPS);
#else
# error "Unsupported platform"
#endif
SectionHeader *sectionHeader = (SectionHeader *)(elfImage + elfHeader->e_shoff);
for(int i = 0; i < elfHeader->e_shnum; i++)
{
if(sectionHeader[i].sh_type == SHT_PROGBITS)
{
if(sectionHeader[i].sh_flags & SHF_EXECINSTR)
{
auto findSectionNameEntryIndex = [&]() -> size_t {
auto sectionNameOffset = sectionHeader[elfHeader->e_shstrndx].sh_offset + sectionHeader[i].sh_name;
const char *sectionName = reinterpret_cast<const char *>(elfImage + sectionNameOffset);
for(size_t j = 0; j < functionNames.size(); ++j)
{
if(strstr(sectionName, functionNames[j]) != nullptr)
{
return j;
}
}
UNREACHABLE("Failed to find executable section that matches input function names");
return static_cast<size_t>(-1);
};
size_t index = findSectionNameEntryIndex();
entryPoints[index].entry = elfImage + sectionHeader[i].sh_offset;
entryPoints[index].codeSize = sectionHeader[i].sh_size;
}
}
else if(sectionHeader[i].sh_type == SHT_REL)
{
ASSERT(sizeof(void *) == 4 && "UNIMPLEMENTED"); // Only expected/implemented for 32-bit code
for(Elf32_Word index = 0; index < sectionHeader[i].sh_size / sectionHeader[i].sh_entsize; index++)
{
const Elf32_Rel &relocation = ((const Elf32_Rel *)(elfImage + sectionHeader[i].sh_offset))[index];
relocateSymbol(elfHeader, relocation, sectionHeader[i]);
}
}
else if(sectionHeader[i].sh_type == SHT_RELA)
{
ASSERT(sizeof(void *) == 8 && "UNIMPLEMENTED"); // Only expected/implemented for 64-bit code
for(Elf32_Word index = 0; index < sectionHeader[i].sh_size / sectionHeader[i].sh_entsize; index++)
{
const Elf64_Rela &relocation = ((const Elf64_Rela *)(elfImage + sectionHeader[i].sh_offset))[index];
relocateSymbol(elfHeader, relocation, sectionHeader[i]);
}
}
}
return entryPoints;
}
template<typename T>
struct ExecutableAllocator
{
ExecutableAllocator() {}
template<class U>
ExecutableAllocator(const ExecutableAllocator<U> &other)
{}
using value_type = T;
using size_type = std::size_t;
T *allocate(size_type n)
{
return (T *)allocateMemoryPages(
sizeof(T) * n, PERMISSION_READ | PERMISSION_WRITE, true);
}
void deallocate(T *p, size_type n)
{
deallocateMemoryPages(p, sizeof(T) * n);
}
};
class ELFMemoryStreamer : public Ice::ELFStreamer, public Routine
{
ELFMemoryStreamer(const ELFMemoryStreamer &) = delete;
ELFMemoryStreamer &operator=(const ELFMemoryStreamer &) = delete;
public:
ELFMemoryStreamer()
: Routine()
{
position = 0;
buffer.reserve(0x1000);
}
~ELFMemoryStreamer() override
{
}
void write8(uint8_t Value) override
{
if(position == (uint64_t)buffer.size())
{
buffer.push_back(Value);
position++;
}
else if(position < (uint64_t)buffer.size())
{
buffer[position] = Value;
position++;
}
else
ASSERT(false && "UNIMPLEMENTED");
}
void writeBytes(llvm::StringRef Bytes) override
{
std::size_t oldSize = buffer.size();
buffer.resize(oldSize + Bytes.size());
memcpy(&buffer[oldSize], Bytes.begin(), Bytes.size());
position += Bytes.size();
}
uint64_t tell() const override
{
return position;
}
void seek(uint64_t Off) override
{
position = Off;
}
std::vector<EntryPoint> loadImageAndGetEntryPoints(const std::vector<const char *> &functionNames)
{
auto entryPoints = loadImage(&buffer[0], functionNames);
#if defined(_WIN32)
FlushInstructionCache(GetCurrentProcess(), NULL, 0);
#else
for(auto &entryPoint : entryPoints)
{
__builtin___clear_cache((char *)entryPoint.entry, (char *)entryPoint.entry + entryPoint.codeSize);
}
#endif
return entryPoints;
}
void finalize()
{
position = std::numeric_limits<std::size_t>::max(); // Can't stream more data after this
protectMemoryPages(&buffer[0], buffer.size(), PERMISSION_READ | PERMISSION_EXECUTE);
}
void setEntry(int index, const void *func)
{
ASSERT(func);
funcs[index] = func;
}
const void *getEntry(int index) const override
{
ASSERT(funcs[index]);
return funcs[index];
}
const void *addConstantData(const void *data, size_t size, size_t alignment = 1)
{
// Check if we already have a suitable constant.
for(const auto &c : constantsPool)
{
void *ptr = c.data.get();
size_t space = c.space;
void *alignedPtr = std::align(alignment, size, ptr, space);
if(space < size)
{
continue;
}
if(memcmp(data, alignedPtr, size) == 0)
{
return alignedPtr;
}
}
// TODO(b/148086935): Replace with a buffer allocator.
size_t space = size + alignment;
auto buf = std::unique_ptr<uint8_t[]>(new uint8_t[space]);
void *ptr = buf.get();
void *alignedPtr = std::align(alignment, size, ptr, space);
ASSERT(alignedPtr);
memcpy(alignedPtr, data, size);
constantsPool.emplace_back(std::move(buf), space);
return alignedPtr;
}
private:
struct Constant
{
Constant(std::unique_ptr<uint8_t[]> data, size_t space)
: data(std::move(data))
, space(space)
{}
std::unique_ptr<uint8_t[]> data;
size_t space;
};
std::array<const void *, Nucleus::CoroutineEntryCount> funcs = {};
std::vector<uint8_t, ExecutableAllocator<uint8_t>> buffer;
std::size_t position;
std::vector<Constant> constantsPool;
};
#ifdef ENABLE_RR_PRINT
void VPrintf(const std::vector<Value *> &vals)
{
sz::Call(::function, ::basicBlock, Ice::IceType_i32, reinterpret_cast<const void *>(rr::DebugPrintf), V(vals), true);
}
#endif // ENABLE_RR_PRINT
Nucleus::Nucleus()
{
::codegenMutex.lock(); // SubzeroReactor is currently not thread safe
Ice::ClFlags &Flags = Ice::ClFlags::Flags;
Ice::ClFlags::getParsedClFlags(Flags);
#if defined(__arm__)
Flags.setTargetArch(Ice::Target_ARM32);
Flags.setTargetInstructionSet(Ice::ARM32InstructionSet_HWDivArm);
#elif defined(__mips__)
Flags.setTargetArch(Ice::Target_MIPS32);
Flags.setTargetInstructionSet(Ice::BaseInstructionSet);
#else // x86
Flags.setTargetArch(sizeof(void *) == 8 ? Ice::Target_X8664 : Ice::Target_X8632);
Flags.setTargetInstructionSet(CPUID::SSE4_1 ? Ice::X86InstructionSet_SSE4_1 : Ice::X86InstructionSet_SSE2);
#endif
Flags.setOutFileType(Ice::FT_Elf);
Flags.setOptLevel(toIce(rr::getPragmaState(rr::OptimizationLevel)));
Flags.setVerbose(subzeroDumpEnabled ? Ice::IceV_Most : Ice::IceV_None);
Flags.setDisableHybridAssembly(true);
// Emit functions into separate sections in the ELF so we can find them by name
Flags.setFunctionSections(true);
static llvm::raw_os_ostream cout(std::cout);
static llvm::raw_os_ostream cerr(std::cerr);
if(subzeroEmitTextAsm)
{
// Decorate text asm with liveness info
Flags.setDecorateAsm(true);
}
if(false) // Write out to a file
{
std::error_code errorCode;
::out = new Ice::Fdstream("out.o", errorCode, llvm::sys::fs::F_None);
::elfFile = new Ice::ELFFileStreamer(*out);
::context = new Ice::GlobalContext(&cout, &cout, &cerr, elfFile);
}
else
{
ELFMemoryStreamer *elfMemory = new ELFMemoryStreamer();
::context = new Ice::GlobalContext(&cout, &cout, &cerr, elfMemory);
::routine = elfMemory;
}
#if !__has_feature(memory_sanitizer)
// thread_local variables in shared libraries are initialized at load-time,
// but this is not observed by MemorySanitizer if the loader itself was not
// instrumented, leading to false-positive uninitialized variable errors.
ASSERT(Variable::unmaterializedVariables == nullptr);
#endif
Variable::unmaterializedVariables = new Variable::UnmaterializedVariables{};
}
Nucleus::~Nucleus()
{
delete Variable::unmaterializedVariables;
Variable::unmaterializedVariables = nullptr;
delete ::routine;
::routine = nullptr;
delete ::allocator;
::allocator = nullptr;
delete ::function;
::function = nullptr;
delete ::context;
::context = nullptr;
delete ::elfFile;
::elfFile = nullptr;
delete ::out;
::out = nullptr;
::entryBlock = nullptr;
::basicBlock = nullptr;
::basicBlockTop = nullptr;
::codegenMutex.unlock();
}
// This function lowers and produces executable binary code in memory for the input functions,
// and returns a Routine with the entry points to these functions.
template<size_t Count>
static std::shared_ptr<Routine> acquireRoutine(Ice::Cfg *const (&functions)[Count], const char *const (&names)[Count])
{
// This logic is modeled after the IceCompiler, as well as GlobalContext::translateFunctions
// and GlobalContext::emitItems.
if(subzeroDumpEnabled)
{
// Output dump strings immediately, rather than once buffer is full. Useful for debugging.
::context->getStrDump().SetUnbuffered();
}
::context->emitFileHeader();
// Translate
for(size_t i = 0; i < Count; ++i)
{
Ice::Cfg *currFunc = functions[i];
// Install function allocator in TLS for Cfg-specific container allocators
Ice::CfgLocalAllocatorScope allocScope(currFunc);
currFunc->setFunctionName(Ice::GlobalString::createWithString(::context, names[i]));
if(::optimizerCallback)
{
Nucleus::OptimizerReport report;
rr::optimize(currFunc, &report);
::optimizerCallback(&report);
::optimizerCallback = nullptr;
}
else
{
rr::optimize(currFunc);
}
currFunc->computeInOutEdges();
ASSERT_MSG(!currFunc->hasError(), "%s", currFunc->getError().c_str());
currFunc->translate();
ASSERT_MSG(!currFunc->hasError(), "%s", currFunc->getError().c_str());
currFunc->getAssembler<>()->setInternal(currFunc->getInternal());
if(subzeroEmitTextAsm)
{
currFunc->emit();
}
currFunc->emitIAS();
if(currFunc->hasError())
{
return nullptr;
}
}
// Emit items
::context->lowerGlobals("");
auto objectWriter = ::context->getObjectWriter();
for(size_t i = 0; i < Count; ++i)
{
Ice::Cfg *currFunc = functions[i];
// Accumulate globals from functions to emit into the "last" section at the end
auto globals = currFunc->getGlobalInits();
if(globals && !globals->empty())
{
::context->getGlobals()->merge(globals.get());
}
auto assembler = currFunc->releaseAssembler();
assembler->alignFunction();
objectWriter->writeFunctionCode(currFunc->getFunctionName(), currFunc->getInternal(), assembler.get());
}
::context->lowerGlobals("last");
::context->lowerConstants();
::context->lowerJumpTables();
objectWriter->setUndefinedSyms(::context->getConstantExternSyms());
::context->emitTargetRODataSections();
objectWriter->writeNonUserSections();
// Done compiling functions, get entry pointers to each of them
auto entryPoints = ::routine->loadImageAndGetEntryPoints({ names, names + Count });
ASSERT(entryPoints.size() == Count);
for(size_t i = 0; i < entryPoints.size(); ++i)
{
::routine->setEntry(i, entryPoints[i].entry);
}
::routine->finalize();
Routine *handoffRoutine = ::routine;
::routine = nullptr;
return std::shared_ptr<Routine>(handoffRoutine);
}
std::shared_ptr<Routine> Nucleus::acquireRoutine(const char *name)
{
finalizeFunction();
return rr::acquireRoutine({ ::function }, { name });
}
Value *Nucleus::allocateStackVariable(Type *t, int arraySize)
{
Ice::Type type = T(t);
int typeSize = Ice::typeWidthInBytes(type);
int totalSize = typeSize * (arraySize ? arraySize : 1);
auto bytes = Ice::ConstantInteger32::create(::context, Ice::IceType_i32, totalSize);
auto address = ::function->makeVariable(T(getPointerType(t)));
auto alloca = Ice::InstAlloca::create(::function, address, bytes, typeSize); // SRoA depends on the alignment to match the type size.
::function->getEntryNode()->getInsts().push_front(alloca);
return V(address);
}
BasicBlock *Nucleus::createBasicBlock()
{
return B(::function->makeNode());
}
BasicBlock *Nucleus::getInsertBlock()
{
return B(::basicBlock);
}
void Nucleus::setInsertBlock(BasicBlock *basicBlock)
{
// ASSERT(::basicBlock->getInsts().back().getTerminatorEdges().size() >= 0 && "Previous basic block must have a terminator");
::basicBlock = basicBlock;
}
void Nucleus::createFunction(Type *returnType, const std::vector<Type *> &paramTypes)
{
ASSERT(::function == nullptr);
ASSERT(::allocator == nullptr);
ASSERT(::entryBlock == nullptr);
ASSERT(::basicBlock == nullptr);
ASSERT(::basicBlockTop == nullptr);
::function = sz::createFunction(::context, T(returnType), T(paramTypes));
// NOTE: The scoped allocator sets the TLS allocator to the one in the function. This global one
// becomes invalid if another one is created; for example, when creating await and destroy functions
// for coroutines, in which case, we must make sure to create a new scoped allocator for ::function again.
// TODO: Get rid of this as a global, and create scoped allocs in every Nucleus function instead.
::allocator = new Ice::CfgLocalAllocatorScope(::function);
::entryBlock = ::function->getEntryNode();
::basicBlock = ::function->makeNode();
::basicBlockTop = ::basicBlock;
}
Value *Nucleus::getArgument(unsigned int index)
{
return V(::function->getArgs()[index]);
}
void Nucleus::createRetVoid()
{
RR_DEBUG_INFO_UPDATE_LOC();
// Code generated after this point is unreachable, so any variables
// being read can safely return an undefined value. We have to avoid
// materializing variables after the terminator ret instruction.
Variable::killUnmaterialized();
Ice::InstRet *ret = Ice::InstRet::create(::function);
::basicBlock->appendInst(ret);
}
void Nucleus::createRet(Value *v)
{
RR_DEBUG_INFO_UPDATE_LOC();
// Code generated after this point is unreachable, so any variables
// being read can safely return an undefined value. We have to avoid
// materializing variables after the terminator ret instruction.
Variable::killUnmaterialized();
Ice::InstRet *ret = Ice::InstRet::create(::function, v);
::basicBlock->appendInst(ret);
}
void Nucleus::createBr(BasicBlock *dest)
{
RR_DEBUG_INFO_UPDATE_LOC();
Variable::materializeAll();
auto br = Ice::InstBr::create(::function, dest);
::basicBlock->appendInst(br);
}
void Nucleus::createCondBr(Value *cond, BasicBlock *ifTrue, BasicBlock *ifFalse)
{
RR_DEBUG_INFO_UPDATE_LOC();
Variable::materializeAll();
auto br = Ice::InstBr::create(::function, cond, ifTrue, ifFalse);
::basicBlock->appendInst(br);
}
static bool isCommutative(Ice::InstArithmetic::OpKind op)
{
switch(op)
{
case Ice::InstArithmetic::Add:
case Ice::InstArithmetic::Fadd:
case Ice::InstArithmetic::Mul:
case Ice::InstArithmetic::Fmul:
case Ice::InstArithmetic::And:
case Ice::InstArithmetic::Or:
case Ice::InstArithmetic::Xor:
return true;
default:
return false;
}
}
static Value *createArithmetic(Ice::InstArithmetic::OpKind op, Value *lhs, Value *rhs)
{
ASSERT(lhs->getType() == rhs->getType() || llvm::isa<Ice::Constant>(rhs));
bool swapOperands = llvm::isa<Ice::Constant>(lhs) && isCommutative(op);
Ice::Variable *result = ::function->makeVariable(lhs->getType());
Ice::InstArithmetic *arithmetic = Ice::InstArithmetic::create(::function, op, result, swapOperands ? rhs : lhs, swapOperands ? lhs : rhs);
::basicBlock->appendInst(arithmetic);
return V(result);
}
Value *Nucleus::createAdd(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createArithmetic(Ice::InstArithmetic::Add, lhs, rhs);
}
Value *Nucleus::createSub(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createArithmetic(Ice::InstArithmetic::Sub, lhs, rhs);
}
Value *Nucleus::createMul(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createArithmetic(Ice::InstArithmetic::Mul, lhs, rhs);
}
Value *Nucleus::createUDiv(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createArithmetic(Ice::InstArithmetic::Udiv, lhs, rhs);
}
Value *Nucleus::createSDiv(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createArithmetic(Ice::InstArithmetic::Sdiv, lhs, rhs);
}
Value *Nucleus::createFAdd(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createArithmetic(Ice::InstArithmetic::Fadd, lhs, rhs);
}
Value *Nucleus::createFSub(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createArithmetic(Ice::InstArithmetic::Fsub, lhs, rhs);
}
Value *Nucleus::createFMul(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createArithmetic(Ice::InstArithmetic::Fmul, lhs, rhs);
}
Value *Nucleus::createFDiv(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createArithmetic(Ice::InstArithmetic::Fdiv, lhs, rhs);
}
Value *Nucleus::createURem(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createArithmetic(Ice::InstArithmetic::Urem, lhs, rhs);
}
Value *Nucleus::createSRem(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createArithmetic(Ice::InstArithmetic::Srem, lhs, rhs);
}
Value *Nucleus::createFRem(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
// TODO(b/148139679) Fix Subzero generating invalid code for FRem on vector types
// createArithmetic(Ice::InstArithmetic::Frem, lhs, rhs);
UNIMPLEMENTED("b/148139679 Nucleus::createFRem");
return nullptr;
}
Value *Nucleus::createShl(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createArithmetic(Ice::InstArithmetic::Shl, lhs, rhs);
}
Value *Nucleus::createLShr(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createArithmetic(Ice::InstArithmetic::Lshr, lhs, rhs);
}
Value *Nucleus::createAShr(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createArithmetic(Ice::InstArithmetic::Ashr, lhs, rhs);
}
Value *Nucleus::createAnd(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createArithmetic(Ice::InstArithmetic::And, lhs, rhs);
}
Value *Nucleus::createOr(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createArithmetic(Ice::InstArithmetic::Or, lhs, rhs);
}
Value *Nucleus::createXor(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createArithmetic(Ice::InstArithmetic::Xor, lhs, rhs);
}
Value *Nucleus::createNeg(Value *v)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createSub(createNullValue(T(v->getType())), v);
}
Value *Nucleus::createFNeg(Value *v)
{
RR_DEBUG_INFO_UPDATE_LOC();
std::vector<double> c = { -0.0 };
Value *negativeZero = Ice::isVectorType(v->getType()) ? createConstantVector(c, T(v->getType())) : V(::context->getConstantFloat(-0.0f));
return createFSub(negativeZero, v);
}
Value *Nucleus::createNot(Value *v)
{
RR_DEBUG_INFO_UPDATE_LOC();
if(Ice::isScalarIntegerType(v->getType()))
{
return createXor(v, V(::context->getConstantInt(v->getType(), -1)));
}
else // Vector
{
std::vector<int64_t> c = { -1 };
return createXor(v, createConstantVector(c, T(v->getType())));
}
}
static void validateAtomicAndMemoryOrderArgs(bool atomic, std::memory_order memoryOrder)
{
#if defined(__i386__) || defined(__x86_64__)
// We're good, atomics and strictest memory order (except seq_cst) are guaranteed.
// Note that sequential memory ordering could be guaranteed by using x86's LOCK prefix.
// Note also that relaxed memory order could be implemented using MOVNTPS and friends.
#else
if(atomic)
{
UNIMPLEMENTED("b/150475088 Atomic load/store not implemented for current platform");
}
if(memoryOrder != std::memory_order_relaxed)
{
UNIMPLEMENTED("b/150475088 Memory order other than memory_order_relaxed not implemented for current platform");
}
#endif
// Vulkan doesn't allow sequential memory order
ASSERT(memoryOrder != std::memory_order_seq_cst);
}
Value *Nucleus::createLoad(Value *ptr, Type *type, bool isVolatile, unsigned int align, bool atomic, std::memory_order memoryOrder)
{
RR_DEBUG_INFO_UPDATE_LOC();
validateAtomicAndMemoryOrderArgs(atomic, memoryOrder);
int valueType = (int)reinterpret_cast<intptr_t>(type);
Ice::Variable *result = nullptr;
if((valueType & EmulatedBits) && (align != 0)) // Narrow vector not stored on stack.
{
if(emulateIntrinsics)
{
if(typeSize(type) == 4)
{
auto pointer = RValue<Pointer<Byte>>(ptr);
Int x = *Pointer<Int>(pointer);
Int4 vector;
vector = Insert(vector, x, 0);
result = ::function->makeVariable(T(type));
auto bitcast = Ice::InstCast::create(::function, Ice::InstCast::Bitcast, result, vector.loadValue());
::basicBlock->appendInst(bitcast);
}
else if(typeSize(type) == 8)
{
ASSERT_MSG(!atomic, "Emulated 64-bit loads are not atomic");
auto pointer = RValue<Pointer<Byte>>(ptr);
Int x = *Pointer<Int>(pointer);
Int y = *Pointer<Int>(pointer + 4);
Int4 vector;
vector = Insert(vector, x, 0);
vector = Insert(vector, y, 1);
result = ::function->makeVariable(T(type));
auto bitcast = Ice::InstCast::create(::function, Ice::InstCast::Bitcast, result, vector.loadValue());
::basicBlock->appendInst(bitcast);
}
else
UNREACHABLE("typeSize(type): %d", int(typeSize(type)));
}
else
{
const Ice::Intrinsics::IntrinsicInfo intrinsic = { Ice::Intrinsics::LoadSubVector, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F };
result = ::function->makeVariable(T(type));
auto load = Ice::InstIntrinsic::create(::function, 2, result, intrinsic);
load->addArg(ptr);
load->addArg(::context->getConstantInt32(typeSize(type)));
::basicBlock->appendInst(load);
}
}
else
{
result = sz::createLoad(::function, ::basicBlock, V(ptr), T(type), align);
}
ASSERT(result);
return V(result);
}
Value *Nucleus::createStore(Value *value, Value *ptr, Type *type, bool isVolatile, unsigned int align, bool atomic, std::memory_order memoryOrder)
{
RR_DEBUG_INFO_UPDATE_LOC();
validateAtomicAndMemoryOrderArgs(atomic, memoryOrder);
#if __has_feature(memory_sanitizer)
// Mark all (non-stack) memory writes as initialized by calling __msan_unpoison
if(align != 0)
{
auto call = Ice::InstCall::create(::function, 2, nullptr, ::context->getConstantInt64(reinterpret_cast<intptr_t>(__msan_unpoison)), false);
call->addArg(ptr);
call->addArg(::context->getConstantInt64(typeSize(type)));
::basicBlock->appendInst(call);
}
#endif
int valueType = (int)reinterpret_cast<intptr_t>(type);
if((valueType & EmulatedBits) && (align != 0)) // Narrow vector not stored on stack.
{
if(emulateIntrinsics)
{
if(typeSize(type) == 4)
{
Ice::Variable *vector = ::function->makeVariable(Ice::IceType_v4i32);
auto bitcast = Ice::InstCast::create(::function, Ice::InstCast::Bitcast, vector, value);
::basicBlock->appendInst(bitcast);
RValue<Int4> v(V(vector));
auto pointer = RValue<Pointer<Byte>>(ptr);
Int x = Extract(v, 0);
*Pointer<Int>(pointer) = x;
}
else if(typeSize(type) == 8)
{
ASSERT_MSG(!atomic, "Emulated 64-bit stores are not atomic");
Ice::Variable *vector = ::function->makeVariable(Ice::IceType_v4i32);
auto bitcast = Ice::InstCast::create(::function, Ice::InstCast::Bitcast, vector, value);
::basicBlock->appendInst(bitcast);
RValue<Int4> v(V(vector));
auto pointer = RValue<Pointer<Byte>>(ptr);
Int x = Extract(v, 0);
*Pointer<Int>(pointer) = x;
Int y = Extract(v, 1);
*Pointer<Int>(pointer + 4) = y;
}
else
UNREACHABLE("typeSize(type): %d", int(typeSize(type)));
}
else
{
const Ice::Intrinsics::IntrinsicInfo intrinsic = { Ice::Intrinsics::StoreSubVector, Ice::Intrinsics::SideEffects_T, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_T };
auto store = Ice::InstIntrinsic::create(::function, 3, nullptr, intrinsic);
store->addArg(value);
store->addArg(ptr);
store->addArg(::context->getConstantInt32(typeSize(type)));
::basicBlock->appendInst(store);
}
}
else
{
ASSERT(value->getType() == T(type));
auto store = Ice::InstStore::create(::function, V(value), V(ptr), align);
::basicBlock->appendInst(store);
}
return value;
}
Value *Nucleus::createGEP(Value *ptr, Type *type, Value *index, bool unsignedIndex)
{
RR_DEBUG_INFO_UPDATE_LOC();
ASSERT(index->getType() == Ice::IceType_i32);
if(auto *constant = llvm::dyn_cast<Ice::ConstantInteger32>(index))
{
int32_t offset = constant->getValue() * (int)typeSize(type);
if(offset == 0)
{
return ptr;
}
return createAdd(ptr, createConstantInt(offset));
}
if(!Ice::isByteSizedType(T(type)))
{
index = createMul(index, createConstantInt((int)typeSize(type)));
}
if(sizeof(void *) == 8)
{
if(unsignedIndex)
{
index = createZExt(index, T(Ice::IceType_i64));
}
else
{
index = createSExt(index, T(Ice::IceType_i64));
}
}
return createAdd(ptr, index);
}
static Value *createAtomicRMW(Ice::Intrinsics::AtomicRMWOperation rmwOp, Value *ptr, Value *value, std::memory_order memoryOrder)
{
Ice::Variable *result = ::function->makeVariable(value->getType());
const Ice::Intrinsics::IntrinsicInfo intrinsic = { Ice::Intrinsics::AtomicRMW, Ice::Intrinsics::SideEffects_T, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_T };
auto inst = Ice::InstIntrinsic::create(::function, 0, result, intrinsic);
auto op = ::context->getConstantInt32(rmwOp);
auto order = ::context->getConstantInt32(stdToIceMemoryOrder(memoryOrder));
inst->addArg(op);
inst->addArg(ptr);
inst->addArg(value);
inst->addArg(order);
::basicBlock->appendInst(inst);
return V(result);
}
Value *Nucleus::createAtomicAdd(Value *ptr, Value *value, std::memory_order memoryOrder)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createAtomicRMW(Ice::Intrinsics::AtomicAdd, ptr, value, memoryOrder);
}
Value *Nucleus::createAtomicSub(Value *ptr, Value *value, std::memory_order memoryOrder)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createAtomicRMW(Ice::Intrinsics::AtomicSub, ptr, value, memoryOrder);
}
Value *Nucleus::createAtomicAnd(Value *ptr, Value *value, std::memory_order memoryOrder)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createAtomicRMW(Ice::Intrinsics::AtomicAnd, ptr, value, memoryOrder);
}
Value *Nucleus::createAtomicOr(Value *ptr, Value *value, std::memory_order memoryOrder)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createAtomicRMW(Ice::Intrinsics::AtomicOr, ptr, value, memoryOrder);
}
Value *Nucleus::createAtomicXor(Value *ptr, Value *value, std::memory_order memoryOrder)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createAtomicRMW(Ice::Intrinsics::AtomicXor, ptr, value, memoryOrder);
}
Value *Nucleus::createAtomicExchange(Value *ptr, Value *value, std::memory_order memoryOrder)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createAtomicRMW(Ice::Intrinsics::AtomicExchange, ptr, value, memoryOrder);
}
Value *Nucleus::createAtomicCompareExchange(Value *ptr, Value *value, Value *compare, std::memory_order memoryOrderEqual, std::memory_order memoryOrderUnequal)
{
RR_DEBUG_INFO_UPDATE_LOC();
Ice::Variable *result = ::function->makeVariable(value->getType());
const Ice::Intrinsics::IntrinsicInfo intrinsic = { Ice::Intrinsics::AtomicCmpxchg, Ice::Intrinsics::SideEffects_T, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_T };
auto inst = Ice::InstIntrinsic::create(::function, 0, result, intrinsic);
auto orderEq = ::context->getConstantInt32(stdToIceMemoryOrder(memoryOrderEqual));
auto orderNeq = ::context->getConstantInt32(stdToIceMemoryOrder(memoryOrderUnequal));
inst->addArg(ptr);
inst->addArg(compare);
inst->addArg(value);
inst->addArg(orderEq);
inst->addArg(orderNeq);
::basicBlock->appendInst(inst);
return V(result);
}
static Value *createCast(Ice::InstCast::OpKind op, Value *v, Type *destType)
{
if(v->getType() == T(destType))
{
return v;
}
Ice::Variable *result = ::function->makeVariable(T(destType));
Ice::InstCast *cast = Ice::InstCast::create(::function, op, result, v);
::basicBlock->appendInst(cast);
return V(result);
}
Value *Nucleus::createTrunc(Value *v, Type *destType)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createCast(Ice::InstCast::Trunc, v, destType);
}
Value *Nucleus::createZExt(Value *v, Type *destType)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createCast(Ice::InstCast::Zext, v, destType);
}
Value *Nucleus::createSExt(Value *v, Type *destType)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createCast(Ice::InstCast::Sext, v, destType);
}
Value *Nucleus::createFPToUI(Value *v, Type *destType)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createCast(Ice::InstCast::Fptoui, v, destType);
}
Value *Nucleus::createFPToSI(Value *v, Type *destType)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createCast(Ice::InstCast::Fptosi, v, destType);
}
Value *Nucleus::createSIToFP(Value *v, Type *destType)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createCast(Ice::InstCast::Sitofp, v, destType);
}
Value *Nucleus::createFPTrunc(Value *v, Type *destType)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createCast(Ice::InstCast::Fptrunc, v, destType);
}
Value *Nucleus::createFPExt(Value *v, Type *destType)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createCast(Ice::InstCast::Fpext, v, destType);
}
Value *Nucleus::createBitCast(Value *v, Type *destType)
{
RR_DEBUG_INFO_UPDATE_LOC();
// Bitcasts must be between types of the same logical size. But with emulated narrow vectors we need
// support for casting between scalars and wide vectors. For platforms where this is not supported,
// emulate them by writing to the stack and reading back as the destination type.
if(emulateMismatchedBitCast || (v->getType() == Ice::Type::IceType_i64))
{
if(!Ice::isVectorType(v->getType()) && Ice::isVectorType(T(destType)))
{
Value *address = allocateStackVariable(destType);
createStore(v, address, T(v->getType()));
return createLoad(address, destType);
}
else if(Ice::isVectorType(v->getType()) && !Ice::isVectorType(T(destType)))
{
Value *address = allocateStackVariable(T(v->getType()));
createStore(v, address, T(v->getType()));
return createLoad(address, destType);
}
}
return createCast(Ice::InstCast::Bitcast, v, destType);
}
static Value *createIntCompare(Ice::InstIcmp::ICond condition, Value *lhs, Value *rhs)
{
ASSERT(lhs->getType() == rhs->getType());
auto result = ::function->makeVariable(Ice::isScalarIntegerType(lhs->getType()) ? Ice::IceType_i1 : lhs->getType());
auto cmp = Ice::InstIcmp::create(::function, condition, result, lhs, rhs);
::basicBlock->appendInst(cmp);
return V(result);
}
Value *Nucleus::createICmpEQ(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createIntCompare(Ice::InstIcmp::Eq, lhs, rhs);
}
Value *Nucleus::createICmpNE(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createIntCompare(Ice::InstIcmp::Ne, lhs, rhs);
}
Value *Nucleus::createICmpUGT(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createIntCompare(Ice::InstIcmp::Ugt, lhs, rhs);
}
Value *Nucleus::createICmpUGE(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createIntCompare(Ice::InstIcmp::Uge, lhs, rhs);
}
Value *Nucleus::createICmpULT(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createIntCompare(Ice::InstIcmp::Ult, lhs, rhs);
}
Value *Nucleus::createICmpULE(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createIntCompare(Ice::InstIcmp::Ule, lhs, rhs);
}
Value *Nucleus::createICmpSGT(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createIntCompare(Ice::InstIcmp::Sgt, lhs, rhs);
}
Value *Nucleus::createICmpSGE(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createIntCompare(Ice::InstIcmp::Sge, lhs, rhs);
}
Value *Nucleus::createICmpSLT(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createIntCompare(Ice::InstIcmp::Slt, lhs, rhs);
}
Value *Nucleus::createICmpSLE(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createIntCompare(Ice::InstIcmp::Sle, lhs, rhs);
}
static Value *createFloatCompare(Ice::InstFcmp::FCond condition, Value *lhs, Value *rhs)
{
ASSERT(lhs->getType() == rhs->getType());
ASSERT(Ice::isScalarFloatingType(lhs->getType()) || lhs->getType() == Ice::IceType_v4f32);
auto result = ::function->makeVariable(Ice::isScalarFloatingType(lhs->getType()) ? Ice::IceType_i1 : Ice::IceType_v4i32);
auto cmp = Ice::InstFcmp::create(::function, condition, result, lhs, rhs);
::basicBlock->appendInst(cmp);
return V(result);
}
Value *Nucleus::createFCmpOEQ(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createFloatCompare(Ice::InstFcmp::Oeq, lhs, rhs);
}
Value *Nucleus::createFCmpOGT(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createFloatCompare(Ice::InstFcmp::Ogt, lhs, rhs);
}
Value *Nucleus::createFCmpOGE(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createFloatCompare(Ice::InstFcmp::Oge, lhs, rhs);
}
Value *Nucleus::createFCmpOLT(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createFloatCompare(Ice::InstFcmp::Olt, lhs, rhs);
}
Value *Nucleus::createFCmpOLE(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createFloatCompare(Ice::InstFcmp::Ole, lhs, rhs);
}
Value *Nucleus::createFCmpONE(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createFloatCompare(Ice::InstFcmp::One, lhs, rhs);
}
Value *Nucleus::createFCmpORD(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createFloatCompare(Ice::InstFcmp::Ord, lhs, rhs);
}
Value *Nucleus::createFCmpUNO(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createFloatCompare(Ice::InstFcmp::Uno, lhs, rhs);
}
Value *Nucleus::createFCmpUEQ(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createFloatCompare(Ice::InstFcmp::Ueq, lhs, rhs);
}
Value *Nucleus::createFCmpUGT(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createFloatCompare(Ice::InstFcmp::Ugt, lhs, rhs);
}
Value *Nucleus::createFCmpUGE(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createFloatCompare(Ice::InstFcmp::Uge, lhs, rhs);
}
Value *Nucleus::createFCmpULT(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createFloatCompare(Ice::InstFcmp::Ult, lhs, rhs);
}
Value *Nucleus::createFCmpULE(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createFloatCompare(Ice::InstFcmp::Ule, lhs, rhs);
}
Value *Nucleus::createFCmpUNE(Value *lhs, Value *rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createFloatCompare(Ice::InstFcmp::Une, lhs, rhs);
}
Value *Nucleus::createExtractElement(Value *vector, Type *type, int index)
{
RR_DEBUG_INFO_UPDATE_LOC();
auto result = ::function->makeVariable(T(type));
auto extract = Ice::InstExtractElement::create(::function, result, V(vector), ::context->getConstantInt32(index));
::basicBlock->appendInst(extract);
return V(result);
}
Value *Nucleus::createInsertElement(Value *vector, Value *element, int index)
{
RR_DEBUG_INFO_UPDATE_LOC();
auto result = ::function->makeVariable(vector->getType());
auto insert = Ice::InstInsertElement::create(::function, result, vector, element, ::context->getConstantInt32(index));
::basicBlock->appendInst(insert);
return V(result);
}
Value *Nucleus::createShuffleVector(Value *V1, Value *V2, std::vector<int> select)
{
RR_DEBUG_INFO_UPDATE_LOC();
ASSERT(V1->getType() == V2->getType());
size_t size = Ice::typeNumElements(V1->getType());
auto result = ::function->makeVariable(V1->getType());
auto shuffle = Ice::InstShuffleVector::create(::function, result, V1, V2);
const size_t selectSize = select.size();
for(size_t i = 0; i < size; i++)
{
shuffle->addIndex(llvm::cast<Ice::ConstantInteger32>(::context->getConstantInt32(select[i % selectSize])));
}
::basicBlock->appendInst(shuffle);
return V(result);
}
Value *Nucleus::createSelect(Value *C, Value *ifTrue, Value *ifFalse)
{
RR_DEBUG_INFO_UPDATE_LOC();
ASSERT(ifTrue->getType() == ifFalse->getType());
auto result = ::function->makeVariable(ifTrue->getType());
auto *select = Ice::InstSelect::create(::function, result, C, ifTrue, ifFalse);
::basicBlock->appendInst(select);
return V(result);
}
SwitchCases *Nucleus::createSwitch(Value *control, BasicBlock *defaultBranch, unsigned numCases)
{
RR_DEBUG_INFO_UPDATE_LOC();
auto switchInst = Ice::InstSwitch::create(::function, numCases, control, defaultBranch);
::basicBlock->appendInst(switchInst);
return reinterpret_cast<SwitchCases *>(switchInst);
}
void Nucleus::addSwitchCase(SwitchCases *switchCases, int label, BasicBlock *branch)
{
RR_DEBUG_INFO_UPDATE_LOC();
switchCases->addBranch(label, label, branch);
}
void Nucleus::createUnreachable()
{
RR_DEBUG_INFO_UPDATE_LOC();
Ice::InstUnreachable *unreachable = Ice::InstUnreachable::create(::function);
::basicBlock->appendInst(unreachable);
}
Type *Nucleus::getType(Value *value)
{
return T(V(value)->getType());
}
Type *Nucleus::getContainedType(Type *vectorType)
{
Ice::Type vecTy = T(vectorType);
switch(vecTy)
{
case Ice::IceType_v4i1: return T(Ice::IceType_i1);
case Ice::IceType_v8i1: return T(Ice::IceType_i1);
case Ice::IceType_v16i1: return T(Ice::IceType_i1);
case Ice::IceType_v16i8: return T(Ice::IceType_i8);
case Ice::IceType_v8i16: return T(Ice::IceType_i16);
case Ice::IceType_v4i32: return T(Ice::IceType_i32);
case Ice::IceType_v4f32: return T(Ice::IceType_f32);
default:
ASSERT_MSG(false, "getContainedType: input type is not a vector type");
return {};
}
}
Type *Nucleus::getPointerType(Type *ElementType)
{
return T(sz::getPointerType(T(ElementType)));
}
static constexpr Ice::Type getNaturalIntType()
{
constexpr size_t intSize = sizeof(int);
static_assert(intSize == 4 || intSize == 8, "");
return intSize == 4 ? Ice::IceType_i32 : Ice::IceType_i64;
}
Type *Nucleus::getPrintfStorageType(Type *valueType)
{
Ice::Type valueTy = T(valueType);
switch(valueTy)
{
case Ice::IceType_i32:
return T(getNaturalIntType());
case Ice::IceType_f32:
return T(Ice::IceType_f64);
default:
UNIMPLEMENTED_NO_BUG("getPrintfStorageType: add more cases as needed");
return {};
}
}
Value *Nucleus::createNullValue(Type *Ty)
{
RR_DEBUG_INFO_UPDATE_LOC();
if(Ice::isVectorType(T(Ty)))
{
ASSERT(Ice::typeNumElements(T(Ty)) <= 16);
std::vector<int64_t> c = { 0 };
return createConstantVector(c, Ty);
}
else
{
return V(::context->getConstantZero(T(Ty)));
}
}
Value *Nucleus::createConstantLong(int64_t i)
{
RR_DEBUG_INFO_UPDATE_LOC();
return V(::context->getConstantInt64(i));
}
Value *Nucleus::createConstantInt(int i)
{
RR_DEBUG_INFO_UPDATE_LOC();
return V(::context->getConstantInt32(i));
}
Value *Nucleus::createConstantInt(unsigned int i)
{
RR_DEBUG_INFO_UPDATE_LOC();
return V(::context->getConstantInt32(i));
}
Value *Nucleus::createConstantBool(bool b)
{
RR_DEBUG_INFO_UPDATE_LOC();
return V(::context->getConstantInt1(b));
}
Value *Nucleus::createConstantByte(signed char i)
{
RR_DEBUG_INFO_UPDATE_LOC();
return V(::context->getConstantInt8(i));
}
Value *Nucleus::createConstantByte(unsigned char i)
{
RR_DEBUG_INFO_UPDATE_LOC();
return V(::context->getConstantInt8(i));
}
Value *Nucleus::createConstantShort(short i)
{
RR_DEBUG_INFO_UPDATE_LOC();
return V(::context->getConstantInt16(i));
}
Value *Nucleus::createConstantShort(unsigned short i)
{
RR_DEBUG_INFO_UPDATE_LOC();
return V(::context->getConstantInt16(i));
}
Value *Nucleus::createConstantFloat(float x)
{
RR_DEBUG_INFO_UPDATE_LOC();
return V(::context->getConstantFloat(x));
}
Value *Nucleus::createNullPointer(Type *Ty)
{
RR_DEBUG_INFO_UPDATE_LOC();
return createNullValue(T(sizeof(void *) == 8 ? Ice::IceType_i64 : Ice::IceType_i32));
}
static Ice::Constant *IceConstantData(const void *data, size_t size, size_t alignment = 1)
{
return sz::getConstantPointer(::context, ::routine->addConstantData(data, size, alignment));
}
Value *Nucleus::createConstantVector(std::vector<int64_t> constants, Type *type)
{
RR_DEBUG_INFO_UPDATE_LOC();
const int vectorSize = 16;
ASSERT(Ice::typeWidthInBytes(T(type)) == vectorSize);
const int alignment = vectorSize;
const auto &i = constants;
const size_t s = constants.size();
// TODO(b/148082873): Fix global variable constants when generating multiple functions
Ice::Constant *ptr = nullptr;
switch((int)reinterpret_cast<intptr_t>(type))
{
case Ice::IceType_v4i32:
case Ice::IceType_v4i1:
{
const int initializer[4] = { (int)i[0 % s], (int)i[1 % s], (int)i[2 % s], (int)i[3 % s] };
static_assert(sizeof(initializer) == vectorSize);
ptr = IceConstantData(initializer, vectorSize, alignment);
}
break;
case Ice::IceType_v8i16:
case Ice::IceType_v8i1:
{
const short initializer[8] = { (short)i[0 % s], (short)i[1 % s], (short)i[2 % s], (short)i[3 % s], (short)i[4 % s], (short)i[5 % s], (short)i[6 % s], (short)i[7 % s] };
static_assert(sizeof(initializer) == vectorSize);
ptr = IceConstantData(initializer, vectorSize, alignment);
}
break;
case Ice::IceType_v16i8:
case Ice::IceType_v16i1:
{
const char initializer[16] = { (char)i[0 % s], (char)i[1 % s], (char)i[2 % s], (char)i[3 % s], (char)i[4 % s], (char)i[5 % s], (char)i[6 % s], (char)i[7 % s],
(char)i[8 % s], (char)i[9 % s], (char)i[10 % s], (char)i[11 % s], (char)i[12 % s], (char)i[13 % s], (char)i[14 % s], (char)i[15 % s] };
static_assert(sizeof(initializer) == vectorSize);
ptr = IceConstantData(initializer, vectorSize, alignment);
}
break;
case Type_v2i32:
{
const int initializer[4] = { (int)i[0 % s], (int)i[1 % s], (int)i[0 % s], (int)i[1 % s] };
static_assert(sizeof(initializer) == vectorSize);
ptr = IceConstantData(initializer, vectorSize, alignment);
}
break;
case Type_v4i16:
{
const short initializer[8] = { (short)i[0 % s], (short)i[1 % s], (short)i[2 % s], (short)i[3 % s], (short)i[0 % s], (short)i[1 % s], (short)i[2 % s], (short)i[3 % s] };
static_assert(sizeof(initializer) == vectorSize);
ptr = IceConstantData(initializer, vectorSize, alignment);
}
break;
case Type_v8i8:
{
const char initializer[16] = { (char)i[0 % s], (char)i[1 % s], (char)i[2 % s], (char)i[3 % s], (char)i[4 % s], (char)i[5 % s], (char)i[6 % s], (char)i[7 % s], (char)i[0 % s], (char)i[1 % s], (char)i[2 % s], (char)i[3 % s], (char)i[4 % s], (char)i[5 % s], (char)i[6 % s], (char)i[7 % s] };
static_assert(sizeof(initializer) == vectorSize);
ptr = IceConstantData(initializer, vectorSize, alignment);
}
break;
case Type_v4i8:
{
const char initializer[16] = { (char)i[0 % s], (char)i[1 % s], (char)i[2 % s], (char)i[3 % s], (char)i[0 % s], (char)i[1 % s], (char)i[2 % s], (char)i[3 % s], (char)i[0 % s], (char)i[1 % s], (char)i[2 % s], (char)i[3 % s], (char)i[0 % s], (char)i[1 % s], (char)i[2 % s], (char)i[3 % s] };
static_assert(sizeof(initializer) == vectorSize);
ptr = IceConstantData(initializer, vectorSize, alignment);
}
break;
default:
UNREACHABLE("Unknown constant vector type: %d", (int)reinterpret_cast<intptr_t>(type));
}
ASSERT(ptr);
Ice::Variable *result = sz::createLoad(::function, ::basicBlock, ptr, T(type), alignment);
return V(result);
}
Value *Nucleus::createConstantVector(std::vector<double> constants, Type *type)
{
RR_DEBUG_INFO_UPDATE_LOC();
const int vectorSize = 16;
ASSERT(Ice::typeWidthInBytes(T(type)) == vectorSize);
const int alignment = vectorSize;
const auto &f = constants;
const size_t s = constants.size();
// TODO(b/148082873): Fix global variable constants when generating multiple functions
Ice::Constant *ptr = nullptr;
switch((int)reinterpret_cast<intptr_t>(type))
{
case Ice::IceType_v4f32:
{
const float initializer[4] = { (float)f[0 % s], (float)f[1 % s], (float)f[2 % s], (float)f[3 % s] };
static_assert(sizeof(initializer) == vectorSize);
ptr = IceConstantData(initializer, vectorSize, alignment);
}
break;
case Type_v2f32:
{
const float initializer[4] = { (float)f[0 % s], (float)f[1 % s], (float)f[0 % s], (float)f[1 % s] };
static_assert(sizeof(initializer) == vectorSize);
ptr = IceConstantData(initializer, vectorSize, alignment);
}
break;
default:
UNREACHABLE("Unknown constant vector type: %d", (int)reinterpret_cast<intptr_t>(type));
}
ASSERT(ptr);
Ice::Variable *result = sz::createLoad(::function, ::basicBlock, ptr, T(type), alignment);
return V(result);
}
Value *Nucleus::createConstantString(const char *v)
{
// NOTE: Do not call RR_DEBUG_INFO_UPDATE_LOC() here to avoid recursion when called from rr::Printv
return V(IceConstantData(v, strlen(v) + 1));
}
void Nucleus::setOptimizerCallback(OptimizerCallback *callback)
{
::optimizerCallback = callback;
}
Type *Void::type()
{
return T(Ice::IceType_void);
}
Type *Bool::type()
{
return T(Ice::IceType_i1);
}
Type *Byte::type()
{
return T(Ice::IceType_i8);
}
Type *SByte::type()
{
return T(Ice::IceType_i8);
}
Type *Short::type()
{
return T(Ice::IceType_i16);
}
Type *UShort::type()
{
return T(Ice::IceType_i16);
}
Type *Byte4::type()
{
return T(Type_v4i8);
}
Type *SByte4::type()
{
return T(Type_v4i8);
}
static RValue<Byte> SaturateUnsigned(RValue<Short> x)
{
return Byte(IfThenElse(Int(x) > 0xFF, Int(0xFF), IfThenElse(Int(x) < 0, Int(0), Int(x))));
}
static RValue<Byte> Extract(RValue<Byte8> val, int i)
{
return RValue<Byte>(Nucleus::createExtractElement(val.value(), Byte::type(), i));
}
static RValue<Byte8> Insert(RValue<Byte8> val, RValue<Byte> element, int i)
{
return RValue<Byte8>(Nucleus::createInsertElement(val.value(), element.value(), i));
}
RValue<Byte8> AddSat(RValue<Byte8> x, RValue<Byte8> y)
{
RR_DEBUG_INFO_UPDATE_LOC();
if(emulateIntrinsics)
{
return Scalarize([](auto a, auto b) { return SaturateUnsigned(Short(Int(a) + Int(b))); }, x, y);
}
else
{
Ice::Variable *result = ::function->makeVariable(Ice::IceType_v16i8);
const Ice::Intrinsics::IntrinsicInfo intrinsic = { Ice::Intrinsics::AddSaturateUnsigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F };
auto paddusb = Ice::InstIntrinsic::create(::function, 2, result, intrinsic);
paddusb->addArg(x.value());
paddusb->addArg(y.value());
::basicBlock->appendInst(paddusb);
return RValue<Byte8>(V(result));
}
}
RValue<Byte8> SubSat(RValue<Byte8> x, RValue<Byte8> y)
{
RR_DEBUG_INFO_UPDATE_LOC();
if(emulateIntrinsics)
{
return Scalarize([](auto a, auto b) { return SaturateUnsigned(Short(Int(a) - Int(b))); }, x, y);
}
else
{
Ice::Variable *result = ::function->makeVariable(Ice::IceType_v16i8);
const Ice::Intrinsics::IntrinsicInfo intrinsic = { Ice::Intrinsics::SubtractSaturateUnsigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F };
auto psubusw = Ice::InstIntrinsic::create(::function, 2, result, intrinsic);
psubusw->addArg(x.value());
psubusw->addArg(y.value());
::basicBlock->appendInst(psubusw);
return RValue<Byte8>(V(result));
}
}
RValue<SByte> Extract(RValue<SByte8> val, int i)
{
RR_DEBUG_INFO_UPDATE_LOC();
return RValue<SByte>(Nucleus::createExtractElement(val.value(), SByte::type(), i));
}
RValue<SByte8> Insert(RValue<SByte8> val, RValue<SByte> element, int i)
{
RR_DEBUG_INFO_UPDATE_LOC();
return RValue<SByte8>(Nucleus::createInsertElement(val.value(), element.value(), i));
}
RValue<SByte8> operator>>(RValue<SByte8> lhs, unsigned char rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
if(emulateIntrinsics)
{
return Scalarize([rhs](auto a) { return a >> SByte(rhs); }, lhs);
}
else
{
#if defined(__i386__) || defined(__x86_64__)
// SSE2 doesn't support byte vector shifts, so shift as shorts and recombine.
RValue<Short4> hi = (As<Short4>(lhs) >> rhs) & Short4(0xFF00u);
RValue<Short4> lo = As<Short4>(As<UShort4>((As<Short4>(lhs) << 8) >> rhs) >> 8);
return As<SByte8>(hi | lo);
#else
return RValue<SByte8>(Nucleus::createAShr(lhs.value(), V(::context->getConstantInt32(rhs))));
#endif
}
}
RValue<Int> SignMask(RValue<Byte8> x)
{
RR_DEBUG_INFO_UPDATE_LOC();
if(emulateIntrinsics || CPUID::ARM)
{
Byte8 xx = As<Byte8>(As<SByte8>(x) >> 7) & Byte8(0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80);
return Int(Extract(xx, 0)) | Int(Extract(xx, 1)) | Int(Extract(xx, 2)) | Int(Extract(xx, 3)) | Int(Extract(xx, 4)) | Int(Extract(xx, 5)) | Int(Extract(xx, 6)) | Int(Extract(xx, 7));
}
else
{
Ice::Variable *result = ::function->makeVariable(Ice::IceType_i32);
const Ice::Intrinsics::IntrinsicInfo intrinsic = { Ice::Intrinsics::SignMask, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F };
auto movmsk = Ice::InstIntrinsic::create(::function, 1, result, intrinsic);
movmsk->addArg(x.value());
::basicBlock->appendInst(movmsk);
return RValue<Int>(V(result)) & 0xFF;
}
}
// RValue<Byte8> CmpGT(RValue<Byte8> x, RValue<Byte8> y)
// {
// return RValue<Byte8>(createIntCompare(Ice::InstIcmp::Ugt, x.value(), y.value()));
// }
RValue<Byte8> CmpEQ(RValue<Byte8> x, RValue<Byte8> y)
{
RR_DEBUG_INFO_UPDATE_LOC();
return RValue<Byte8>(Nucleus::createICmpEQ(x.value(), y.value()));
}
Type *Byte8::type()
{
return T(Type_v8i8);
}
// RValue<SByte8> operator<<(RValue<SByte8> lhs, unsigned char rhs)
// {
// return RValue<SByte8>(Nucleus::createShl(lhs.value(), V(::context->getConstantInt32(rhs))));
// }
// RValue<SByte8> operator>>(RValue<SByte8> lhs, unsigned char rhs)
// {
// return RValue<SByte8>(Nucleus::createAShr(lhs.value(), V(::context->getConstantInt32(rhs))));
// }
RValue<SByte> SaturateSigned(RValue<Short> x)
{
RR_DEBUG_INFO_UPDATE_LOC();
return SByte(IfThenElse(Int(x) > 0x7F, Int(0x7F), IfThenElse(Int(x) < -0x80, Int(0x80), Int(x))));
}
RValue<SByte8> AddSat(RValue<SByte8> x, RValue<SByte8> y)
{
RR_DEBUG_INFO_UPDATE_LOC();
if(emulateIntrinsics)
{
return Scalarize([](auto a, auto b) { return SaturateSigned(Short(Int(a) + Int(b))); }, x, y);
}
else
{
Ice::Variable *result = ::function->makeVariable(Ice::IceType_v16i8);
const Ice::Intrinsics::IntrinsicInfo intrinsic = { Ice::Intrinsics::AddSaturateSigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F };
auto paddsb = Ice::InstIntrinsic::create(::function, 2, result, intrinsic);
paddsb->addArg(x.value());
paddsb->addArg(y.value());
::basicBlock->appendInst(paddsb);
return RValue<SByte8>(V(result));
}
}
RValue<SByte8> SubSat(RValue<SByte8> x, RValue<SByte8> y)
{
RR_DEBUG_INFO_UPDATE_LOC();
if(emulateIntrinsics)
{
return Scalarize([](auto a, auto b) { return SaturateSigned(Short(Int(a) - Int(b))); }, x, y);
}
else
{
Ice::Variable *result = ::function->makeVariable(Ice::IceType_v16i8);
const Ice::Intrinsics::IntrinsicInfo intrinsic = { Ice::Intrinsics::SubtractSaturateSigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F };
auto psubsb = Ice::InstIntrinsic::create(::function, 2, result, intrinsic);
psubsb->addArg(x.value());
psubsb->addArg(y.value());
::basicBlock->appendInst(psubsb);
return RValue<SByte8>(V(result));
}
}
RValue<Int> SignMask(RValue<SByte8> x)
{
RR_DEBUG_INFO_UPDATE_LOC();
if(emulateIntrinsics || CPUID::ARM)
{
SByte8 xx = (x >> 7) & SByte8(0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80);
return Int(Extract(xx, 0)) | Int(Extract(xx, 1)) | Int(Extract(xx, 2)) | Int(Extract(xx, 3)) | Int(Extract(xx, 4)) | Int(Extract(xx, 5)) | Int(Extract(xx, 6)) | Int(Extract(xx, 7));
}
else
{
Ice::Variable *result = ::function->makeVariable(Ice::IceType_i32);
const Ice::Intrinsics::IntrinsicInfo intrinsic = { Ice::Intrinsics::SignMask, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F };
auto movmsk = Ice::InstIntrinsic::create(::function, 1, result, intrinsic);
movmsk->addArg(x.value());
::basicBlock->appendInst(movmsk);
return RValue<Int>(V(result)) & 0xFF;
}
}
RValue<Byte8> CmpGT(RValue<SByte8> x, RValue<SByte8> y)
{
RR_DEBUG_INFO_UPDATE_LOC();
return RValue<Byte8>(createIntCompare(Ice::InstIcmp::Sgt, x.value(), y.value()));
}
RValue<Byte8> CmpEQ(RValue<SByte8> x, RValue<SByte8> y)
{
RR_DEBUG_INFO_UPDATE_LOC();
return RValue<Byte8>(Nucleus::createICmpEQ(x.value(), y.value()));
}
Type *SByte8::type()
{
return T(Type_v8i8);
}
Type *Byte16::type()
{
return T(Ice::IceType_v16i8);
}
Type *SByte16::type()
{
return T(Ice::IceType_v16i8);
}
Type *Short2::type()
{
return T(Type_v2i16);
}
Type *UShort2::type()
{
return T(Type_v2i16);
}
Short4::Short4(RValue<Int4> cast)
{
std::vector<int> select = { 0, 2, 4, 6, 0, 2, 4, 6 };
Value *short8 = Nucleus::createBitCast(cast.value(), Short8::type());
Value *packed = Nucleus::createShuffleVector(short8, short8, select);
Value *int2 = RValue<Int2>(Int2(As<Int4>(packed))).value();
Value *short4 = Nucleus::createBitCast(int2, Short4::type());
storeValue(short4);
}
// Short4::Short4(RValue<Float> cast)
// {
// }
Short4::Short4(RValue<Float4> cast)
{
// TODO(b/150791192): Generalize and optimize
auto smin = std::numeric_limits<short>::min();
auto smax = std::numeric_limits<short>::max();
*this = Short4(Int4(Max(Min(cast, Float4(smax)), Float4(smin))));
}
RValue<Short4> operator<<(RValue<Short4> lhs, unsigned char rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
if(emulateIntrinsics)
{
return Scalarize([rhs](auto x) { return x << Short(rhs); }, lhs);
}
else
{
return RValue<Short4>(Nucleus::createShl(lhs.value(), V(::context->getConstantInt32(rhs))));
}
}
RValue<Short4> operator>>(RValue<Short4> lhs, unsigned char rhs)
{
RR_DEBUG_INFO_UPDATE_LOC();
if(emulateIntrinsics)
{
return Scalarize([rhs](auto x) { return x >> Short(rhs); }, lhs);
}
else
{
return RValue<Short4>(Nucleus::createAShr(lhs.value(), V(::context->getConstantInt32(rhs))));
}
}
RValue<Short4> Max(RValue<Short4> x, RValue<Short4> y)
{
RR_DEBUG_INFO_UPDATE_LOC();
Ice::Variable *condition = ::function->makeVariable(Ice::IceType_v8i1);
auto cmp = Ice::InstIcmp::create(::function, Ice::InstIcmp::Sle, condition, x.value(), y.value());
::basicBlock->appendInst(cmp);
Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
auto select = Ice::InstSelect::create(::function, result, condition, y.value(), x.value());
::basicBlock->appendInst(select);
return RValue<Short4>(V(result));
}
RValue<Short4> Min(RValue<Short4> x, RValue<Short4> y)
{
RR_DEBUG_INFO_UPDATE_LOC();
Ice::Variable *condition = ::function->makeVariable(Ice::IceType_v8i1);
auto cmp = Ice::InstIcmp::create(::function, Ice::InstIcmp::Sgt, condition, x.value(), y.value());
::basicBlock->appendInst(cmp);
Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
auto select = Ice::InstSelect::create(::function, result, condition, y.value(), x.value());
::basicBlock->appendInst(select);
return RValue<Short4>(V(result));
}
RValue<Short> SaturateSigned(RValue<Int> x)
{
RR_DEBUG_INFO_UPDATE_LOC();
return Short(IfThenElse(x > 0x7FFF, Int(0x7FFF), IfThenElse(x < -0x8000, Int(0x8000), x)));
}
RValue<Short4> AddSat(RValue<Short4> x, RValue<Short4> y)
{
RR_DEBUG_INFO_UPDATE_LOC();
if(emulateIntrinsics)
{
return Scalarize([](auto a, auto b) { return SaturateSigned(Int(a) + Int(b)); }, x, y);
}
else
{
Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
const Ice::Intrinsics::IntrinsicInfo intrinsic = { Ice::Intrinsics::AddSaturateSigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F };
auto paddsw = Ice::InstIntrinsic::create(::function, 2, result, intrinsic);
paddsw->addArg(x.value());
paddsw->addArg(y.value());
::basicBlock->appendInst(paddsw);
return RValue<Short4>(V(result));
}
}
RValue<Short4> SubSat(RValue<Short4> x, RValue<Short4> y)
{
RR_DEBUG_INFO_UPDATE_LOC();
if(emulateIntrinsics)
{
return Scalarize([](auto a, auto b) { return SaturateSigned(Int(a) - Int(b)); }, x, y);
}
else
{
Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
const Ice::Intrinsics::IntrinsicInfo intrinsic = { Ice::Intrinsics::SubtractSaturateSigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F };
auto psubsw = Ice::InstIntrinsic::create(::function, 2, result, intrinsic);
psubsw->addArg(x.value());
psubsw->addArg(y.value());
::basicBlock->appendInst(psubsw);
return RValue<Short4>(V(result));
}
}
RValue<Short4> MulHigh(RValue<Short4> x, RValue<Short4> y)
{
RR_DEBUG_INFO_UPDATE_LOC();
if(emulateIntrinsics)
{
return Scalarize([](auto a, auto b) { return Short((Int(a) * Int(b)) >> 16); }, x, y);
}
else
{
Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
const Ice::Intrinsics::IntrinsicInfo intrinsic = { Ice::Intrinsics::MultiplyHighSigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F };
auto pmulhw = Ice::InstIntrinsic::create(::function, 2, result, intrinsic);
pmulhw->addArg(x.value());
pmulhw->addArg(y.value());
::basicBlock->appendInst(pmulhw);
return RValue<Short4>(V(result));
}
}
RValue<Int2> MulAdd(RValue<Short4> x, RValue<Short4> y)
{
RR_DEBUG_INFO_UPDATE_LOC();
if(emulateIntrinsics)
{
Int2 result;
result = Insert(result, Int(Extract(x, 0)) * Int(Extract(y, 0)) + Int(Extract(x, 1)) * Int(Extract(y, 1)), 0);
result = Insert(result, Int(Extract(x, 2)) * Int(Extract(y, 2)) + Int(Extract(x, 3)) * Int(Extract(y, 3)), 1);
return result;
}
else
{
Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
const Ice::Intrinsics::IntrinsicInfo intrinsic = { Ice::Intrinsics::MultiplyAddPairs, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F };
auto pmaddwd = Ice::InstIntrinsic::create(::function, 2, result, intrinsic);
pmaddwd->addArg(x.value());
pmaddwd->addArg(y.value());
::basicBlock->appendInst(pmaddwd);
return As<Int2>(V(result));
}
}
RValue<SByte8> PackSigned(RValue<Short4> x, RValue<Short4> y)
{
RR_DEBUG_INFO_UPDATE_LOC();
if(emulateIntrinsics)
{
SByte8 result;
result = Insert(result, SaturateSigned(Extract(x, 0)), 0);
result = Insert(result, SaturateSigned(Extract(x, 1)), 1);
result = Insert(result, SaturateSigned(Extract(x, 2)), 2);
result = Insert(result, SaturateSigned(Extract(x, 3)), 3);
result = Insert(result, SaturateSigned(Extract(y, 0)), 4);
result = Insert(result, SaturateSigned(Extract(y, 1)), 5);
result = Insert(result, SaturateSigned(Extract(y, 2)), 6);
result = Insert(result, SaturateSigned(Extract(y, 3)), 7);
return result;
}
else
{
Ice::Variable *result = ::function->makeVariable(Ice::IceType_v16i8);
const Ice::Intrinsics::IntrinsicInfo intrinsic = { Ice::Intrinsics::VectorPackSigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F };
auto pack = Ice::InstIntrinsic::create(::function, 2, result, intrinsic);
pack->addArg(x.value());