Google Git Google Git
Sign in
swiftshader/SwiftShader/b8f1a3a/./third_party/llvm-10.0/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
blob: 5b177c0c5d9d5166d0414df328dae068b3fe2c79 [file]
//=- WebAssemblyISelLowering.cpp - WebAssembly DAG Lowering Implementation -==//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
/// This file implements the WebAssemblyTargetLowering class.
///
//===----------------------------------------------------------------------===//
#include "WebAssemblyISelLowering.h"
#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
#include "WebAssemblyMachineFunctionInfo.h"
#include "WebAssemblySubtarget.h"
#include "WebAssemblyTargetMachine.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/CallingConvLower.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/WasmEHFuncInfo.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/DiagnosticPrinter.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsWebAssembly.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetOptions.h"
using namespace llvm;
#define DEBUG_TYPE "wasm-lower"
WebAssemblyTargetLowering::WebAssemblyTargetLowering(
const TargetMachine &TM, const WebAssemblySubtarget &STI)
: TargetLowering(TM), Subtarget(&STI) {
auto MVTPtr = Subtarget->hasAddr64() ? MVT::i64 : MVT::i32;
// Booleans always contain 0 or 1.
setBooleanContents(ZeroOrOneBooleanContent);
// Except in SIMD vectors
setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
// We don't know the microarchitecture here, so just reduce register pressure.
setSchedulingPreference(Sched::RegPressure);
// Tell ISel that we have a stack pointer.
setStackPointerRegisterToSaveRestore(
Subtarget->hasAddr64() ? WebAssembly::SP64 : WebAssembly::SP32);
// Set up the register classes.
addRegisterClass(MVT::i32, &WebAssembly::I32RegClass);
addRegisterClass(MVT::i64, &WebAssembly::I64RegClass);
addRegisterClass(MVT::f32, &WebAssembly::F32RegClass);
addRegisterClass(MVT::f64, &WebAssembly::F64RegClass);
if (Subtarget->hasSIMD128()) {
addRegisterClass(MVT::v16i8, &WebAssembly::V128RegClass);
addRegisterClass(MVT::v8i16, &WebAssembly::V128RegClass);
addRegisterClass(MVT::v4i32, &WebAssembly::V128RegClass);
addRegisterClass(MVT::v4f32, &WebAssembly::V128RegClass);
}
if (Subtarget->hasUnimplementedSIMD128()) {
addRegisterClass(MVT::v2i64, &WebAssembly::V128RegClass);
addRegisterClass(MVT::v2f64, &WebAssembly::V128RegClass);
}
// Compute derived properties from the register classes.
computeRegisterProperties(Subtarget->getRegisterInfo());
setOperationAction(ISD::GlobalAddress, MVTPtr, Custom);
setOperationAction(ISD::ExternalSymbol, MVTPtr, Custom);
setOperationAction(ISD::JumpTable, MVTPtr, Custom);
setOperationAction(ISD::BlockAddress, MVTPtr, Custom);
setOperationAction(ISD::BRIND, MVT::Other, Custom);
// Take the default expansion for va_arg, va_copy, and va_end. There is no
// default action for va_start, so we do that custom.
setOperationAction(ISD::VASTART, MVT::Other, Custom);
setOperationAction(ISD::VAARG, MVT::Other, Expand);
setOperationAction(ISD::VACOPY, MVT::Other, Expand);
setOperationAction(ISD::VAEND, MVT::Other, Expand);
for (auto T : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) {
// Don't expand the floating-point types to constant pools.
setOperationAction(ISD::ConstantFP, T, Legal);
// Expand floating-point comparisons.
for (auto CC : {ISD::SETO, ISD::SETUO, ISD::SETUEQ, ISD::SETONE,
ISD::SETULT, ISD::SETULE, ISD::SETUGT, ISD::SETUGE})
setCondCodeAction(CC, T, Expand);
// Expand floating-point library function operators.
for (auto Op :
{ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW, ISD::FREM, ISD::FMA})
setOperationAction(Op, T, Expand);
// Note supported floating-point library function operators that otherwise
// default to expand.
for (auto Op :
{ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FNEARBYINT, ISD::FRINT})
setOperationAction(Op, T, Legal);
// Support minimum and maximum, which otherwise default to expand.
setOperationAction(ISD::FMINIMUM, T, Legal);
setOperationAction(ISD::FMAXIMUM, T, Legal);
// WebAssembly currently has no builtin f16 support.
setOperationAction(ISD::FP16_TO_FP, T, Expand);
setOperationAction(ISD::FP_TO_FP16, T, Expand);
setLoadExtAction(ISD::EXTLOAD, T, MVT::f16, Expand);
setTruncStoreAction(T, MVT::f16, Expand);
}
// Expand unavailable integer operations.
for (auto Op :
{ISD::BSWAP, ISD::SMUL_LOHI, ISD::UMUL_LOHI, ISD::MULHS, ISD::MULHU,
ISD::SDIVREM, ISD::UDIVREM, ISD::SHL_PARTS, ISD::SRA_PARTS,
ISD::SRL_PARTS, ISD::ADDC, ISD::ADDE, ISD::SUBC, ISD::SUBE}) {
for (auto T : {MVT::i32, MVT::i64})
setOperationAction(Op, T, Expand);
if (Subtarget->hasSIMD128())
for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
setOperationAction(Op, T, Expand);
if (Subtarget->hasUnimplementedSIMD128())
setOperationAction(Op, MVT::v2i64, Expand);
}
// SIMD-specific configuration
if (Subtarget->hasSIMD128()) {
// Support saturating add for i8x16 and i16x8
for (auto Op : {ISD::SADDSAT, ISD::UADDSAT})
for (auto T : {MVT::v16i8, MVT::v8i16})
setOperationAction(Op, T, Legal);
// Custom lower BUILD_VECTORs to minimize number of replace_lanes
for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32})
setOperationAction(ISD::BUILD_VECTOR, T, Custom);
if (Subtarget->hasUnimplementedSIMD128())
for (auto T : {MVT::v2i64, MVT::v2f64})
setOperationAction(ISD::BUILD_VECTOR, T, Custom);
// We have custom shuffle lowering to expose the shuffle mask
for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32})
setOperationAction(ISD::VECTOR_SHUFFLE, T, Custom);
if (Subtarget->hasUnimplementedSIMD128())
for (auto T: {MVT::v2i64, MVT::v2f64})
setOperationAction(ISD::VECTOR_SHUFFLE, T, Custom);
// Custom lowering since wasm shifts must have a scalar shift amount
for (auto Op : {ISD::SHL, ISD::SRA, ISD::SRL}) {
for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
setOperationAction(Op, T, Custom);
if (Subtarget->hasUnimplementedSIMD128())
setOperationAction(Op, MVT::v2i64, Custom);
}
// Custom lower lane accesses to expand out variable indices
for (auto Op : {ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT}) {
for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32})
setOperationAction(Op, T, Custom);
if (Subtarget->hasUnimplementedSIMD128())
for (auto T : {MVT::v2i64, MVT::v2f64})
setOperationAction(Op, T, Custom);
}
// There is no i64x2.mul instruction
setOperationAction(ISD::MUL, MVT::v2i64, Expand);
// There are no vector select instructions
for (auto Op : {ISD::VSELECT, ISD::SELECT_CC, ISD::SELECT}) {
for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32})
setOperationAction(Op, T, Expand);
if (Subtarget->hasUnimplementedSIMD128())
for (auto T : {MVT::v2i64, MVT::v2f64})
setOperationAction(Op, T, Expand);
}
// Expand integer operations supported for scalars but not SIMD
for (auto Op : {ISD::CTLZ, ISD::CTTZ, ISD::CTPOP, ISD::SDIV, ISD::UDIV,
ISD::SREM, ISD::UREM, ISD::ROTL, ISD::ROTR}) {
for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
setOperationAction(Op, T, Expand);
if (Subtarget->hasUnimplementedSIMD128())
setOperationAction(Op, MVT::v2i64, Expand);
}
// But we do have integer min and max operations
if (Subtarget->hasUnimplementedSIMD128()) {
for (auto Op : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX})
for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
setOperationAction(Op, T, Legal);
}
// Expand float operations supported for scalars but not SIMD
for (auto Op : {ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FNEARBYINT,
ISD::FCOPYSIGN, ISD::FLOG, ISD::FLOG2, ISD::FLOG10,
ISD::FEXP, ISD::FEXP2, ISD::FRINT}) {
setOperationAction(Op, MVT::v4f32, Expand);
if (Subtarget->hasUnimplementedSIMD128())
setOperationAction(Op, MVT::v2f64, Expand);
}
// Expand operations not supported for i64x2 vectors
if (Subtarget->hasUnimplementedSIMD128())
for (unsigned CC = 0; CC < ISD::SETCC_INVALID; ++CC)
setCondCodeAction(static_cast<ISD::CondCode>(CC), MVT::v2i64, Custom);
// Expand additional SIMD ops that V8 hasn't implemented yet
if (!Subtarget->hasUnimplementedSIMD128()) {
setOperationAction(ISD::FSQRT, MVT::v4f32, Expand);
setOperationAction(ISD::FDIV, MVT::v4f32, Expand);
}
}
// As a special case, these operators use the type to mean the type to
// sign-extend from.
setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
if (!Subtarget->hasSignExt()) {
// Sign extends are legal only when extending a vector extract
auto Action = Subtarget->hasSIMD128() ? Custom : Expand;
for (auto T : {MVT::i8, MVT::i16, MVT::i32})
setOperationAction(ISD::SIGN_EXTEND_INREG, T, Action);
}
for (auto T : MVT::integer_fixedlen_vector_valuetypes())
setOperationAction(ISD::SIGN_EXTEND_INREG, T, Expand);
// Dynamic stack allocation: use the default expansion.
setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
setOperationAction(ISD::DYNAMIC_STACKALLOC, MVTPtr, Expand);
setOperationAction(ISD::FrameIndex, MVT::i32, Custom);
setOperationAction(ISD::CopyToReg, MVT::Other, Custom);
// Expand these forms; we pattern-match the forms that we can handle in isel.
for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64})
for (auto Op : {ISD::BR_CC, ISD::SELECT_CC})
setOperationAction(Op, T, Expand);
// We have custom switch handling.
setOperationAction(ISD::BR_JT, MVT::Other, Custom);
// WebAssembly doesn't have:
// - Floating-point extending loads.
// - Floating-point truncating stores.
// - i1 extending loads.
// - truncating SIMD stores and most extending loads
setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
setTruncStoreAction(MVT::f64, MVT::f32, Expand);
for (auto T : MVT::integer_valuetypes())
for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})
setLoadExtAction(Ext, T, MVT::i1, Promote);
if (Subtarget->hasSIMD128()) {
for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v4f32,
MVT::v2f64}) {
for (auto MemT : MVT::fixedlen_vector_valuetypes()) {
if (MVT(T) != MemT) {
setTruncStoreAction(T, MemT, Expand);
for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})
setLoadExtAction(Ext, T, MemT, Expand);
}
}
}
// But some vector extending loads are legal
if (Subtarget->hasUnimplementedSIMD128()) {
for (auto Ext : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}) {
setLoadExtAction(Ext, MVT::v8i16, MVT::v8i8, Legal);
setLoadExtAction(Ext, MVT::v4i32, MVT::v4i16, Legal);
setLoadExtAction(Ext, MVT::v2i64, MVT::v2i32, Legal);
}
}
}
// Don't do anything clever with build_pairs
setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand);
// Trap lowers to wasm unreachable
setOperationAction(ISD::TRAP, MVT::Other, Legal);
// Exception handling intrinsics
setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
setMaxAtomicSizeInBitsSupported(64);
// Override the __gnu_f2h_ieee/__gnu_h2f_ieee names so that the f32 name is
// consistent with the f64 and f128 names.
setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
// Define the emscripten name for return address helper.
// TODO: when implementing other WASM backends, make this generic or only do
// this on emscripten depending on what they end up doing.
setLibcallName(RTLIB::RETURN_ADDRESS, "emscripten_return_address");
// Always convert switches to br_tables unless there is only one case, which
// is equivalent to a simple branch. This reduces code size for wasm, and we
// defer possible jump table optimizations to the VM.
setMinimumJumpTableEntries(2);
}
TargetLowering::AtomicExpansionKind
WebAssemblyTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
// We have wasm instructions for these
switch (AI->getOperation()) {
case AtomicRMWInst::Add:
case AtomicRMWInst::Sub:
case AtomicRMWInst::And:
case AtomicRMWInst::Or:
case AtomicRMWInst::Xor:
case AtomicRMWInst::Xchg:
return AtomicExpansionKind::None;
default:
break;
}
return AtomicExpansionKind::CmpXChg;
}
FastISel *WebAssemblyTargetLowering::createFastISel(
FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo) const {
return WebAssembly::createFastISel(FuncInfo, LibInfo);
}
MVT WebAssemblyTargetLowering::getScalarShiftAmountTy(const DataLayout & /*DL*/,
EVT VT) const {
unsigned BitWidth = NextPowerOf2(VT.getSizeInBits() - 1);
if (BitWidth > 1 && BitWidth < 8)
BitWidth = 8;
if (BitWidth > 64) {
// The shift will be lowered to a libcall, and compiler-rt libcalls expect
// the count to be an i32.
BitWidth = 32;
assert(BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) &&
"32-bit shift counts ought to be enough for anyone");
}
MVT Result = MVT::getIntegerVT(BitWidth);
assert(Result != MVT::INVALID_SIMPLE_VALUE_TYPE &&
"Unable to represent scalar shift amount type");
return Result;
}
// Lower an fp-to-int conversion operator from the LLVM opcode, which has an
// undefined result on invalid/overflow, to the WebAssembly opcode, which
// traps on invalid/overflow.
static MachineBasicBlock *LowerFPToInt(MachineInstr &MI, DebugLoc DL,
MachineBasicBlock *BB,
const TargetInstrInfo &TII,
bool IsUnsigned, bool Int64,
bool Float64, unsigned LoweredOpcode) {
MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
Register OutReg = MI.getOperand(0).getReg();
Register InReg = MI.getOperand(1).getReg();
unsigned Abs = Float64 ? WebAssembly::ABS_F64 : WebAssembly::ABS_F32;
unsigned FConst = Float64 ? WebAssembly::CONST_F64 : WebAssembly::CONST_F32;
unsigned LT = Float64 ? WebAssembly::LT_F64 : WebAssembly::LT_F32;
unsigned GE = Float64 ? WebAssembly::GE_F64 : WebAssembly::GE_F32;
unsigned IConst = Int64 ? WebAssembly::CONST_I64 : WebAssembly::CONST_I32;
unsigned Eqz = WebAssembly::EQZ_I32;
unsigned And = WebAssembly::AND_I32;
int64_t Limit = Int64 ? INT64_MIN : INT32_MIN;
int64_t Substitute = IsUnsigned ? 0 : Limit;
double CmpVal = IsUnsigned ? -(double)Limit * 2.0 : -(double)Limit;
auto &Context = BB->getParent()->getFunction().getContext();
Type *Ty = Float64 ? Type::getDoubleTy(Context) : Type::getFloatTy(Context);
const BasicBlock *LLVMBB = BB->getBasicBlock();
MachineFunction *F = BB->getParent();
MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(LLVMBB);
MachineBasicBlock *FalseMBB = F->CreateMachineBasicBlock(LLVMBB);
MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(LLVMBB);
MachineFunction::iterator It = ++BB->getIterator();
F->insert(It, FalseMBB);
F->insert(It, TrueMBB);
F->insert(It, DoneMBB);
// Transfer the remainder of BB and its successor edges to DoneMBB.
DoneMBB->splice(DoneMBB->begin(), BB, std::next(MI.getIterator()), BB->end());
DoneMBB->transferSuccessorsAndUpdatePHIs(BB);
BB->addSuccessor(TrueMBB);
BB->addSuccessor(FalseMBB);
TrueMBB->addSuccessor(DoneMBB);
FalseMBB->addSuccessor(DoneMBB);
unsigned Tmp0, Tmp1, CmpReg, EqzReg, FalseReg, TrueReg;
Tmp0 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
CmpReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
EqzReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
FalseReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg));
TrueReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg));
MI.eraseFromParent();
// For signed numbers, we can do a single comparison to determine whether
// fabs(x) is within range.
if (IsUnsigned) {
Tmp0 = InReg;
} else {
BuildMI(BB, DL, TII.get(Abs), Tmp0).addReg(InReg);
}
BuildMI(BB, DL, TII.get(FConst), Tmp1)
.addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, CmpVal)));
BuildMI(BB, DL, TII.get(LT), CmpReg).addReg(Tmp0).addReg(Tmp1);
// For unsigned numbers, we have to do a separate comparison with zero.
if (IsUnsigned) {
Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
Register SecondCmpReg =
MRI.createVirtualRegister(&WebAssembly::I32RegClass);
Register AndReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
BuildMI(BB, DL, TII.get(FConst), Tmp1)
.addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, 0.0)));
BuildMI(BB, DL, TII.get(GE), SecondCmpReg).addReg(Tmp0).addReg(Tmp1);
BuildMI(BB, DL, TII.get(And), AndReg).addReg(CmpReg).addReg(SecondCmpReg);
CmpReg = AndReg;
}
BuildMI(BB, DL, TII.get(Eqz), EqzReg).addReg(CmpReg);
// Create the CFG diamond to select between doing the conversion or using
// the substitute value.
BuildMI(BB, DL, TII.get(WebAssembly::BR_IF)).addMBB(TrueMBB).addReg(EqzReg);
BuildMI(FalseMBB, DL, TII.get(LoweredOpcode), FalseReg).addReg(InReg);
BuildMI(FalseMBB, DL, TII.get(WebAssembly::BR)).addMBB(DoneMBB);
BuildMI(TrueMBB, DL, TII.get(IConst), TrueReg).addImm(Substitute);
BuildMI(*DoneMBB, DoneMBB->begin(), DL, TII.get(TargetOpcode::PHI), OutReg)
.addReg(FalseReg)
.addMBB(FalseMBB)
.addReg(TrueReg)
.addMBB(TrueMBB);
return DoneMBB;
}
MachineBasicBlock *WebAssemblyTargetLowering::EmitInstrWithCustomInserter(
MachineInstr &MI, MachineBasicBlock *BB) const {
const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
DebugLoc DL = MI.getDebugLoc();
switch (MI.getOpcode()) {
default:
llvm_unreachable("Unexpected instr type to insert");
case WebAssembly::FP_TO_SINT_I32_F32:
return LowerFPToInt(MI, DL, BB, TII, false, false, false,
WebAssembly::I32_TRUNC_S_F32);
case WebAssembly::FP_TO_UINT_I32_F32:
return LowerFPToInt(MI, DL, BB, TII, true, false, false,
WebAssembly::I32_TRUNC_U_F32);
case WebAssembly::FP_TO_SINT_I64_F32:
return LowerFPToInt(MI, DL, BB, TII, false, true, false,
WebAssembly::I64_TRUNC_S_F32);
case WebAssembly::FP_TO_UINT_I64_F32:
return LowerFPToInt(MI, DL, BB, TII, true, true, false,
WebAssembly::I64_TRUNC_U_F32);
case WebAssembly::FP_TO_SINT_I32_F64:
return LowerFPToInt(MI, DL, BB, TII, false, false, true,
WebAssembly::I32_TRUNC_S_F64);
case WebAssembly::FP_TO_UINT_I32_F64:
return LowerFPToInt(MI, DL, BB, TII, true, false, true,
WebAssembly::I32_TRUNC_U_F64);
case WebAssembly::FP_TO_SINT_I64_F64:
return LowerFPToInt(MI, DL, BB, TII, false, true, true,
WebAssembly::I64_TRUNC_S_F64);
case WebAssembly::FP_TO_UINT_I64_F64:
return LowerFPToInt(MI, DL, BB, TII, true, true, true,
WebAssembly::I64_TRUNC_U_F64);
llvm_unreachable("Unexpected instruction to emit with custom inserter");
}
}
const char *
WebAssemblyTargetLowering::getTargetNodeName(unsigned Opcode) const {
switch (static_cast<WebAssemblyISD::NodeType>(Opcode)) {
case WebAssemblyISD::FIRST_NUMBER:
case WebAssemblyISD::FIRST_MEM_OPCODE:
break;
#define HANDLE_NODETYPE(NODE) \
case WebAssemblyISD::NODE: \
return "WebAssemblyISD::" #NODE;
#define HANDLE_MEM_NODETYPE(NODE) HANDLE_NODETYPE(NODE)
#include "WebAssemblyISD.def"
#undef HANDLE_MEM_NODETYPE
#undef HANDLE_NODETYPE
}
return nullptr;
}
std::pair<unsigned, const TargetRegisterClass *>
WebAssemblyTargetLowering::getRegForInlineAsmConstraint(
const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
// First, see if this is a constraint that directly corresponds to a
// WebAssembly register class.
if (Constraint.size() == 1) {
switch (Constraint[0]) {
case 'r':
assert(VT != MVT::iPTR && "Pointer MVT not expected here");
if (Subtarget->hasSIMD128() && VT.isVector()) {
if (VT.getSizeInBits() == 128)
return std::make_pair(0U, &WebAssembly::V128RegClass);
}
if (VT.isInteger() && !VT.isVector()) {
if (VT.getSizeInBits() <= 32)
return std::make_pair(0U, &WebAssembly::I32RegClass);
if (VT.getSizeInBits() <= 64)
return std::make_pair(0U, &WebAssembly::I64RegClass);
}
break;
default:
break;
}
}
return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
}
bool WebAssemblyTargetLowering::isCheapToSpeculateCttz() const {
// Assume ctz is a relatively cheap operation.
return true;
}
bool WebAssemblyTargetLowering::isCheapToSpeculateCtlz() const {
// Assume clz is a relatively cheap operation.
return true;
}
bool WebAssemblyTargetLowering::isLegalAddressingMode(const DataLayout &DL,
const AddrMode &AM,
Type *Ty, unsigned AS,
Instruction *I) const {
// WebAssembly offsets are added as unsigned without wrapping. The
// isLegalAddressingMode gives us no way to determine if wrapping could be
// happening, so we approximate this by accepting only non-negative offsets.
if (AM.BaseOffs < 0)
return false;
// WebAssembly has no scale register operands.
if (AM.Scale != 0)
return false;
// Everything else is legal.
return true;
}
bool WebAssemblyTargetLowering::allowsMisalignedMemoryAccesses(
EVT /*VT*/, unsigned /*AddrSpace*/, unsigned /*Align*/,
MachineMemOperand::Flags /*Flags*/, bool *Fast) const {
// WebAssembly supports unaligned accesses, though it should be declared
// with the p2align attribute on loads and stores which do so, and there
// may be a performance impact. We tell LLVM they're "fast" because
// for the kinds of things that LLVM uses this for (merging adjacent stores
// of constants, etc.), WebAssembly implementations will either want the
// unaligned access or they'll split anyway.
if (Fast)
*Fast = true;
return true;
}
bool WebAssemblyTargetLowering::isIntDivCheap(EVT VT,
AttributeList Attr) const {
// The current thinking is that wasm engines will perform this optimization,
// so we can save on code size.
return true;
}
bool WebAssemblyTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
if (!Subtarget->hasUnimplementedSIMD128())
return false;
MVT ExtT = ExtVal.getSimpleValueType();
MVT MemT = cast<LoadSDNode>(ExtVal->getOperand(0))->getSimpleValueType(0);
return (ExtT == MVT::v8i16 && MemT == MVT::v8i8) ||
(ExtT == MVT::v4i32 && MemT == MVT::v4i16) ||
(ExtT == MVT::v2i64 && MemT == MVT::v2i32);
}
EVT WebAssemblyTargetLowering::getSetCCResultType(const DataLayout &DL,
LLVMContext &C,
EVT VT) const {
if (VT.isVector())
return VT.changeVectorElementTypeToInteger();
return TargetLowering::getSetCCResultType(DL, C, VT);
}
bool WebAssemblyTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
const CallInst &I,
MachineFunction &MF,
unsigned Intrinsic) const {
switch (Intrinsic) {
case Intrinsic::wasm_atomic_notify:
Info.opc = ISD::INTRINSIC_W_CHAIN;
Info.memVT = MVT::i32;
Info.ptrVal = I.getArgOperand(0);
Info.offset = 0;
Info.align = Align(4);
// atomic.notify instruction does not really load the memory specified with
// this argument, but MachineMemOperand should either be load or store, so
// we set this to a load.
// FIXME Volatile isn't really correct, but currently all LLVM atomic
// instructions are treated as volatiles in the backend, so we should be
// consistent. The same applies for wasm_atomic_wait intrinsics too.
Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
return true;
case Intrinsic::wasm_atomic_wait_i32:
Info.opc = ISD::INTRINSIC_W_CHAIN;
Info.memVT = MVT::i32;
Info.ptrVal = I.getArgOperand(0);
Info.offset = 0;
Info.align = Align(4);
Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
return true;
case Intrinsic::wasm_atomic_wait_i64:
Info.opc = ISD::INTRINSIC_W_CHAIN;
Info.memVT = MVT::i64;
Info.ptrVal = I.getArgOperand(0);
Info.offset = 0;
Info.align = Align(8);
Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
return true;
default:
return false;
}
}
//===----------------------------------------------------------------------===//
// WebAssembly Lowering private implementation.
//===----------------------------------------------------------------------===//
//===----------------------------------------------------------------------===//
// Lowering Code
//===----------------------------------------------------------------------===//
static void fail(const SDLoc &DL, SelectionDAG &DAG, const char *Msg) {
MachineFunction &MF = DAG.getMachineFunction();
DAG.getContext()->diagnose(
DiagnosticInfoUnsupported(MF.getFunction(), Msg, DL.getDebugLoc()));
}
// Test whether the given calling convention is supported.
static bool callingConvSupported(CallingConv::ID CallConv) {
// We currently support the language-independent target-independent
// conventions. We don't yet have a way to annotate calls with properties like
// "cold", and we don't have any call-clobbered registers, so these are mostly
// all handled the same.
return CallConv == CallingConv::C || CallConv == CallingConv::Fast ||
CallConv == CallingConv::Cold ||
CallConv == CallingConv::PreserveMost ||
CallConv == CallingConv::PreserveAll ||
CallConv == CallingConv::CXX_FAST_TLS ||
CallConv == CallingConv::WASM_EmscriptenInvoke;
}
SDValue
WebAssemblyTargetLowering::LowerCall(CallLoweringInfo &CLI,
SmallVectorImpl<SDValue> &InVals) const {
SelectionDAG &DAG = CLI.DAG;
SDLoc DL = CLI.DL;
SDValue Chain = CLI.Chain;
SDValue Callee = CLI.Callee;
MachineFunction &MF = DAG.getMachineFunction();
auto Layout = MF.getDataLayout();
CallingConv::ID CallConv = CLI.CallConv;
if (!callingConvSupported(CallConv))
fail(DL, DAG,
"WebAssembly doesn't support language-specific or target-specific "
"calling conventions yet");
if (CLI.IsPatchPoint)
fail(DL, DAG, "WebAssembly doesn't support patch point yet");
if (CLI.IsTailCall) {
bool MustTail = CLI.CS && CLI.CS.isMustTailCall();
if (Subtarget->hasTailCall() && !CLI.IsVarArg) {
// Do not tail call unless caller and callee return types match
const Function &F = MF.getFunction();
const TargetMachine &TM = getTargetMachine();
Type *RetTy = F.getReturnType();
SmallVector<MVT, 4> CallerRetTys;
SmallVector<MVT, 4> CalleeRetTys;
computeLegalValueVTs(F, TM, RetTy, CallerRetTys);
computeLegalValueVTs(F, TM, CLI.RetTy, CalleeRetTys);
bool TypesMatch = CallerRetTys.size() == CalleeRetTys.size() &&
std::equal(CallerRetTys.begin(), CallerRetTys.end(),
CalleeRetTys.begin());
if (!TypesMatch) {
// musttail in this case would be an LLVM IR validation failure
assert(!MustTail);
CLI.IsTailCall = false;
}
} else {
CLI.IsTailCall = false;
if (MustTail) {
if (CLI.IsVarArg) {
// The return would pop the argument buffer
fail(DL, DAG, "WebAssembly does not support varargs tail calls");
} else {
fail(DL, DAG, "WebAssembly 'tail-call' feature not enabled");
}
}
}
}
SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
if (Ins.size() > 1)
fail(DL, DAG, "WebAssembly doesn't support more than 1 returned value yet");
SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
// The generic code may have added an sret argument. If we're lowering an
// invoke function, the ABI requires that the function pointer be the first
// argument, so we may have to swap the arguments.
if (CallConv == CallingConv::WASM_EmscriptenInvoke && Outs.size() >= 2 &&
Outs[0].Flags.isSRet()) {
std::swap(Outs[0], Outs[1]);
std::swap(OutVals[0], OutVals[1]);
}
unsigned NumFixedArgs = 0;
for (unsigned I = 0; I < Outs.size(); ++I) {
const ISD::OutputArg &Out = Outs[I];
SDValue &OutVal = OutVals[I];
if (Out.Flags.isNest())
fail(DL, DAG, "WebAssembly hasn't implemented nest arguments");
if (Out.Flags.isInAlloca())
fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments");
if (Out.Flags.isInConsecutiveRegs())
fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments");
if (Out.Flags.isInConsecutiveRegsLast())
fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments");
if (Out.Flags.isByVal() && Out.Flags.getByValSize() != 0) {
auto &MFI = MF.getFrameInfo();
int FI = MFI.CreateStackObject(Out.Flags.getByValSize(),
Out.Flags.getByValAlign(),
/*isSS=*/false);
SDValue SizeNode =
DAG.getConstant(Out.Flags.getByValSize(), DL, MVT::i32);
SDValue FINode = DAG.getFrameIndex(FI, getPointerTy(Layout));
Chain = DAG.getMemcpy(
Chain, DL, FINode, OutVal, SizeNode, Out.Flags.getByValAlign(),
/*isVolatile*/ false, /*AlwaysInline=*/false,
/*isTailCall*/ false, MachinePointerInfo(), MachinePointerInfo());
OutVal = FINode;
}
// Count the number of fixed args *after* legalization.
NumFixedArgs += Out.IsFixed;
}
bool IsVarArg = CLI.IsVarArg;
auto PtrVT = getPointerTy(Layout);
// Analyze operands of the call, assigning locations to each operand.
SmallVector<CCValAssign, 16> ArgLocs;
CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
if (IsVarArg) {
// Outgoing non-fixed arguments are placed in a buffer. First
// compute their offsets and the total amount of buffer space needed.
for (unsigned I = NumFixedArgs; I < Outs.size(); ++I) {
const ISD::OutputArg &Out = Outs[I];
SDValue &Arg = OutVals[I];
EVT VT = Arg.getValueType();
assert(VT != MVT::iPTR && "Legalized args should be concrete");
Type *Ty = VT.getTypeForEVT(*DAG.getContext());
unsigned Align = std::max(Out.Flags.getOrigAlign(),
Layout.getABITypeAlignment(Ty));
unsigned Offset = CCInfo.AllocateStack(Layout.getTypeAllocSize(Ty),
Align);
CCInfo.addLoc(CCValAssign::getMem(ArgLocs.size(), VT.getSimpleVT(),
Offset, VT.getSimpleVT(),
CCValAssign::Full));
}
}
unsigned NumBytes = CCInfo.getAlignedCallFrameSize();
SDValue FINode;
if (IsVarArg && NumBytes) {
// For non-fixed arguments, next emit stores to store the argument values
// to the stack buffer at the offsets computed above.
int FI = MF.getFrameInfo().CreateStackObject(NumBytes,
Layout.getStackAlignment(),
/*isSS=*/false);
unsigned ValNo = 0;
SmallVector<SDValue, 8> Chains;
for (SDValue Arg :
make_range(OutVals.begin() + NumFixedArgs, OutVals.end())) {
assert(ArgLocs[ValNo].getValNo() == ValNo &&
"ArgLocs should remain in order and only hold varargs args");
unsigned Offset = ArgLocs[ValNo++].getLocMemOffset();
FINode = DAG.getFrameIndex(FI, getPointerTy(Layout));
SDValue Add = DAG.getNode(ISD::ADD, DL, PtrVT, FINode,
DAG.getConstant(Offset, DL, PtrVT));
Chains.push_back(
DAG.getStore(Chain, DL, Arg, Add,
MachinePointerInfo::getFixedStack(MF, FI, Offset), 0));
}
if (!Chains.empty())
Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
} else if (IsVarArg) {
FINode = DAG.getIntPtrConstant(0, DL);
}
if (Callee->getOpcode() == ISD::GlobalAddress) {
// If the callee is a GlobalAddress node (quite common, every direct call
// is) turn it into a TargetGlobalAddress node so that LowerGlobalAddress
// doesn't at MO_GOT which is not needed for direct calls.
GlobalAddressSDNode* GA = cast<GlobalAddressSDNode>(Callee);
Callee = DAG.getTargetGlobalAddress(GA->getGlobal(), DL,
getPointerTy(DAG.getDataLayout()),
GA->getOffset());
Callee = DAG.getNode(WebAssemblyISD::Wrapper, DL,
getPointerTy(DAG.getDataLayout()), Callee);
}
// Compute the operands for the CALLn node.
SmallVector<SDValue, 16> Ops;
Ops.push_back(Chain);
Ops.push_back(Callee);
// Add all fixed arguments. Note that for non-varargs calls, NumFixedArgs
// isn't reliable.
Ops.append(OutVals.begin(),
IsVarArg ? OutVals.begin() + NumFixedArgs : OutVals.end());
// Add a pointer to the vararg buffer.
if (IsVarArg)
Ops.push_back(FINode);
SmallVector<EVT, 8> InTys;
for (const auto &In : Ins) {
assert(!In.Flags.isByVal() && "byval is not valid for return values");
assert(!In.Flags.isNest() && "nest is not valid for return values");
if (In.Flags.isInAlloca())
fail(DL, DAG, "WebAssembly hasn't implemented inalloca return values");
if (In.Flags.isInConsecutiveRegs())
fail(DL, DAG, "WebAssembly hasn't implemented cons regs return values");
if (In.Flags.isInConsecutiveRegsLast())
fail(DL, DAG,
"WebAssembly hasn't implemented cons regs last return values");
// Ignore In.getOrigAlign() because all our arguments are passed in
// registers.
InTys.push_back(In.VT);
}
if (CLI.IsTailCall) {
// ret_calls do not return values to the current frame
SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
return DAG.getNode(WebAssemblyISD::RET_CALL, DL, NodeTys, Ops);
}
InTys.push_back(MVT::Other);
SDVTList InTyList = DAG.getVTList(InTys);
SDValue Res =
DAG.getNode(Ins.empty() ? WebAssemblyISD::CALL0 : WebAssemblyISD::CALL1,
DL, InTyList, Ops);
if (Ins.empty()) {
Chain = Res;
} else {
InVals.push_back(Res);
Chain = Res.getValue(1);
}
return Chain;
}
bool WebAssemblyTargetLowering::CanLowerReturn(
CallingConv::ID /*CallConv*/, MachineFunction & /*MF*/, bool /*IsVarArg*/,
const SmallVectorImpl<ISD::OutputArg> &Outs,
LLVMContext & /*Context*/) const {
// WebAssembly can only handle returning tuples with multivalue enabled
return Subtarget->hasMultivalue() || Outs.size() <= 1;
}
SDValue WebAssemblyTargetLowering::LowerReturn(
SDValue Chain, CallingConv::ID CallConv, bool /*IsVarArg*/,
const SmallVectorImpl<ISD::OutputArg> &Outs,
const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
SelectionDAG &DAG) const {
assert((Subtarget->hasMultivalue() || Outs.size() <= 1) &&
"MVP WebAssembly can only return up to one value");
if (!callingConvSupported(CallConv))
fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions");
SmallVector<SDValue, 4> RetOps(1, Chain);
RetOps.append(OutVals.begin(), OutVals.end());
Chain = DAG.getNode(WebAssemblyISD::RETURN, DL, MVT::Other, RetOps);
// Record the number and types of the return values.
for (const ISD::OutputArg &Out : Outs) {
assert(!Out.Flags.isByVal() && "byval is not valid for return values");
assert(!Out.Flags.isNest() && "nest is not valid for return values");
assert(Out.IsFixed && "non-fixed return value is not valid");
if (Out.Flags.isInAlloca())
fail(DL, DAG, "WebAssembly hasn't implemented inalloca results");
if (Out.Flags.isInConsecutiveRegs())
fail(DL, DAG, "WebAssembly hasn't implemented cons regs results");
if (Out.Flags.isInConsecutiveRegsLast())
fail(DL, DAG, "WebAssembly hasn't implemented cons regs last results");
}
return Chain;
}
SDValue WebAssemblyTargetLowering::LowerFormalArguments(
SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
if (!callingConvSupported(CallConv))
fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions");
MachineFunction &MF = DAG.getMachineFunction();
auto *MFI = MF.getInfo<WebAssemblyFunctionInfo>();
// Set up the incoming ARGUMENTS value, which serves to represent the liveness
// of the incoming values before they're represented by virtual registers.
MF.getRegInfo().addLiveIn(WebAssembly::ARGUMENTS);
for (const ISD::InputArg &In : Ins) {
if (In.Flags.isInAlloca())
fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments");
if (In.Flags.isNest())
fail(DL, DAG, "WebAssembly hasn't implemented nest arguments");
if (In.Flags.isInConsecutiveRegs())
fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments");
if (In.Flags.isInConsecutiveRegsLast())
fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments");
// Ignore In.getOrigAlign() because all our arguments are passed in
// registers.
InVals.push_back(In.Used ? DAG.getNode(WebAssemblyISD::ARGUMENT, DL, In.VT,
DAG.getTargetConstant(InVals.size(),
DL, MVT::i32))
: DAG.getUNDEF(In.VT));
// Record the number and types of arguments.
MFI->addParam(In.VT);
}
// Varargs are copied into a buffer allocated by the caller, and a pointer to
// the buffer is passed as an argument.
if (IsVarArg) {
MVT PtrVT = getPointerTy(MF.getDataLayout());
Register VarargVreg =
MF.getRegInfo().createVirtualRegister(getRegClassFor(PtrVT));
MFI->setVarargBufferVreg(VarargVreg);
Chain = DAG.getCopyToReg(
Chain, DL, VarargVreg,
DAG.getNode(WebAssemblyISD::ARGUMENT, DL, PtrVT,
DAG.getTargetConstant(Ins.size(), DL, MVT::i32)));
MFI->addParam(PtrVT);
}
// Record the number and types of arguments and results.
SmallVector<MVT, 4> Params;
SmallVector<MVT, 4> Results;
computeSignatureVTs(MF.getFunction().getFunctionType(), MF.getFunction(),
DAG.getTarget(), Params, Results);
for (MVT VT : Results)
MFI->addResult(VT);
// TODO: Use signatures in WebAssemblyMachineFunctionInfo too and unify
// the param logic here with ComputeSignatureVTs
assert(MFI->getParams().size() == Params.size() &&
std::equal(MFI->getParams().begin(), MFI->getParams().end(),
Params.begin()));
return Chain;
}
void WebAssemblyTargetLowering::ReplaceNodeResults(
SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
switch (N->getOpcode()) {
case ISD::SIGN_EXTEND_INREG:
// Do not add any results, signifying that N should not be custom lowered
// after all. This happens because simd128 turns on custom lowering for
// SIGN_EXTEND_INREG, but for non-vector sign extends the result might be an
// illegal type.
break;
default:
llvm_unreachable(
"ReplaceNodeResults not implemented for this op for WebAssembly!");
}
}
//===----------------------------------------------------------------------===//
// Custom lowering hooks.
//===----------------------------------------------------------------------===//
SDValue WebAssemblyTargetLowering::LowerOperation(SDValue Op,
SelectionDAG &DAG) const {
SDLoc DL(Op);
switch (Op.getOpcode()) {
default:
llvm_unreachable("unimplemented operation lowering");
return SDValue();
case ISD::FrameIndex:
return LowerFrameIndex(Op, DAG);
case ISD::GlobalAddress:
return LowerGlobalAddress(Op, DAG);
case ISD::ExternalSymbol:
return LowerExternalSymbol(Op, DAG);
case ISD::JumpTable:
return LowerJumpTable(Op, DAG);
case ISD::BR_JT:
return LowerBR_JT(Op, DAG);
case ISD::VASTART:
return LowerVASTART(Op, DAG);
case ISD::BlockAddress:
case ISD::BRIND:
fail(DL, DAG, "WebAssembly hasn't implemented computed gotos");
return SDValue();
case ISD::RETURNADDR:
return LowerRETURNADDR(Op, DAG);
case ISD::FRAMEADDR:
return LowerFRAMEADDR(Op, DAG);
case ISD::CopyToReg:
return LowerCopyToReg(Op, DAG);
case ISD::EXTRACT_VECTOR_ELT:
case ISD::INSERT_VECTOR_ELT:
return LowerAccessVectorElement(Op, DAG);
case ISD::INTRINSIC_VOID:
case ISD::INTRINSIC_WO_CHAIN:
case ISD::INTRINSIC_W_CHAIN:
return LowerIntrinsic(Op, DAG);
case ISD::SIGN_EXTEND_INREG:
return LowerSIGN_EXTEND_INREG(Op, DAG);
case ISD::BUILD_VECTOR:
return LowerBUILD_VECTOR(Op, DAG);
case ISD::VECTOR_SHUFFLE:
return LowerVECTOR_SHUFFLE(Op, DAG);
case ISD::SETCC:
return LowerSETCC(Op, DAG);
case ISD::SHL:
case ISD::SRA:
case ISD::SRL:
return LowerShift(Op, DAG);
}
}
SDValue WebAssemblyTargetLowering::LowerCopyToReg(SDValue Op,
SelectionDAG &DAG) const {
SDValue Src = Op.getOperand(2);
if (isa<FrameIndexSDNode>(Src.getNode())) {
// CopyToReg nodes don't support FrameIndex operands. Other targets select
// the FI to some LEA-like instruction, but since we don't have that, we
// need to insert some kind of instruction that can take an FI operand and
// produces a value usable by CopyToReg (i.e. in a vreg). So insert a dummy
// local.copy between Op and its FI operand.
SDValue Chain = Op.getOperand(0);
SDLoc DL(Op);
unsigned Reg = cast<RegisterSDNode>(Op.getOperand(1))->getReg();
EVT VT = Src.getValueType();
SDValue Copy(DAG.getMachineNode(VT == MVT::i32 ? WebAssembly::COPY_I32
: WebAssembly::COPY_I64,
DL, VT, Src),
0);
return Op.getNode()->getNumValues() == 1
? DAG.getCopyToReg(Chain, DL, Reg, Copy)
: DAG.getCopyToReg(Chain, DL, Reg, Copy,
Op.getNumOperands() == 4 ? Op.getOperand(3)
: SDValue());
}
return SDValue();
}
SDValue WebAssemblyTargetLowering::LowerFrameIndex(SDValue Op,
SelectionDAG &DAG) const {
int FI = cast<FrameIndexSDNode>(Op)->getIndex();
return DAG.getTargetFrameIndex(FI, Op.getValueType());
}
SDValue WebAssemblyTargetLowering::LowerRETURNADDR(SDValue Op,
SelectionDAG &DAG) const {
SDLoc DL(Op);
if (!Subtarget->getTargetTriple().isOSEmscripten()) {
fail(DL, DAG,
"Non-Emscripten WebAssembly hasn't implemented "
"__builtin_return_address");
return SDValue();
}
if (verifyReturnAddressArgumentIsConstant(Op, DAG))
return SDValue();
unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
MakeLibCallOptions CallOptions;
return makeLibCall(DAG, RTLIB::RETURN_ADDRESS, Op.getValueType(),
{DAG.getConstant(Depth, DL, MVT::i32)}, CallOptions, DL)
.first;
}
SDValue WebAssemblyTargetLowering::LowerFRAMEADDR(SDValue Op,
SelectionDAG &DAG) const {
// Non-zero depths are not supported by WebAssembly currently. Use the
// legalizer's default expansion, which is to return 0 (what this function is
// documented to do).
if (Op.getConstantOperandVal(0) > 0)
return SDValue();
DAG.getMachineFunction().getFrameInfo().setFrameAddressIsTaken(true);
EVT VT = Op.getValueType();
Register FP =
Subtarget->getRegisterInfo()->getFrameRegister(DAG.getMachineFunction());
return DAG.getCopyFromReg(DAG.getEntryNode(), SDLoc(Op), FP, VT);
}
SDValue WebAssemblyTargetLowering::LowerGlobalAddress(SDValue Op,
SelectionDAG &DAG) const {
SDLoc DL(Op);
const auto *GA = cast<GlobalAddressSDNode>(Op);
EVT VT = Op.getValueType();
assert(GA->getTargetFlags() == 0 &&
"Unexpected target flags on generic GlobalAddressSDNode");
if (GA->getAddressSpace() != 0)
fail(DL, DAG, "WebAssembly only expects the 0 address space");
unsigned OperandFlags = 0;
if (isPositionIndependent()) {
const GlobalValue *GV = GA->getGlobal();
if (getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV)) {
MachineFunction &MF = DAG.getMachineFunction();
MVT PtrVT = getPointerTy(MF.getDataLayout());
const char *BaseName;
if (GV->getValueType()->isFunctionTy()) {
BaseName = MF.createExternalSymbolName("__table_base");
OperandFlags = WebAssemblyII::MO_TABLE_BASE_REL;
}
else {
BaseName = MF.createExternalSymbolName("__memory_base");
OperandFlags = WebAssemblyII::MO_MEMORY_BASE_REL;
}
SDValue BaseAddr =
DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT,
DAG.getTargetExternalSymbol(BaseName, PtrVT));
SDValue SymAddr = DAG.getNode(
WebAssemblyISD::WrapperPIC, DL, VT,
DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT, GA->getOffset(),
OperandFlags));
return DAG.getNode(ISD::ADD, DL, VT, BaseAddr, SymAddr);
} else {
OperandFlags = WebAssemblyII::MO_GOT;
}
}
return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT,
GA->getOffset(), OperandFlags));
}
SDValue
WebAssemblyTargetLowering::LowerExternalSymbol(SDValue Op,
SelectionDAG &DAG) const {
SDLoc DL(Op);
const auto *ES = cast<ExternalSymbolSDNode>(Op);
EVT VT = Op.getValueType();
assert(ES->getTargetFlags() == 0 &&
"Unexpected target flags on generic ExternalSymbolSDNode");
return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
DAG.getTargetExternalSymbol(ES->getSymbol(), VT));
}
SDValue WebAssemblyTargetLowering::LowerJumpTable(SDValue Op,
SelectionDAG &DAG) const {
// There's no need for a Wrapper node because we always incorporate a jump
// table operand into a BR_TABLE instruction, rather than ever
// materializing it in a register.
const JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
return DAG.getTargetJumpTable(JT->getIndex(), Op.getValueType(),
JT->getTargetFlags());
}
SDValue WebAssemblyTargetLowering::LowerBR_JT(SDValue Op,
SelectionDAG &DAG) const {
SDLoc DL(Op);
SDValue Chain = Op.getOperand(0);
const auto *JT = cast<JumpTableSDNode>(Op.getOperand(1));
SDValue Index = Op.getOperand(2);
assert(JT->getTargetFlags() == 0 && "WebAssembly doesn't set target flags");
SmallVector<SDValue, 8> Ops;
Ops.push_back(Chain);
Ops.push_back(Index);
MachineJumpTableInfo *MJTI = DAG.getMachineFunction().getJumpTableInfo();
const auto &MBBs = MJTI->getJumpTables()[JT->getIndex()].MBBs;
// Add an operand for each case.
for (auto MBB : MBBs)
Ops.push_back(DAG.getBasicBlock(MBB));
// TODO: For now, we just pick something arbitrary for a default case for now.
// We really want to sniff out the guard and put in the real default case (and
// delete the guard).
Ops.push_back(DAG.getBasicBlock(MBBs[0]));
return DAG.getNode(WebAssemblyISD::BR_TABLE, DL, MVT::Other, Ops);
}
SDValue WebAssemblyTargetLowering::LowerVASTART(SDValue Op,
SelectionDAG &DAG) const {
SDLoc DL(Op);
EVT PtrVT = getPointerTy(DAG.getMachineFunction().getDataLayout());
auto *MFI = DAG.getMachineFunction().getInfo<WebAssemblyFunctionInfo>();
const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
SDValue ArgN = DAG.getCopyFromReg(DAG.getEntryNode(), DL,
MFI->getVarargBufferVreg(), PtrVT);
return DAG.getStore(Op.getOperand(0), DL, ArgN, Op.getOperand(1),
MachinePointerInfo(SV), 0);
}
SDValue WebAssemblyTargetLowering::LowerIntrinsic(SDValue Op,
SelectionDAG &DAG) const {
MachineFunction &MF = DAG.getMachineFunction();
unsigned IntNo;
switch (Op.getOpcode()) {
case ISD::INTRINSIC_VOID:
case ISD::INTRINSIC_W_CHAIN:
IntNo = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
break;
case ISD::INTRINSIC_WO_CHAIN:
IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
break;
default:
llvm_unreachable("Invalid intrinsic");
}
SDLoc DL(Op);
switch (IntNo) {
default:
return SDValue(); // Don't custom lower most intrinsics.
case Intrinsic::wasm_lsda: {
EVT VT = Op.getValueType();
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
auto &Context = MF.getMMI().getContext();
MCSymbol *S = Context.getOrCreateSymbol(Twine("GCC_except_table") +
Twine(MF.getFunctionNumber()));
return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
DAG.getMCSymbol(S, PtrVT));
}
case Intrinsic::wasm_throw: {
// We only support C++ exceptions for now
int Tag = cast<ConstantSDNode>(Op.getOperand(2).getNode())->getZExtValue();
if (Tag != CPP_EXCEPTION)
llvm_unreachable("Invalid tag!");
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
const char *SymName = MF.createExternalSymbolName("__cpp_exception");
SDValue SymNode = DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT,
DAG.getTargetExternalSymbol(SymName, PtrVT));
return DAG.getNode(WebAssemblyISD::THROW, DL,
MVT::Other, // outchain type
{
Op.getOperand(0), // inchain
SymNode, // exception symbol
Op.getOperand(3) // thrown value
});
}
}
}
SDValue
WebAssemblyTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
SelectionDAG &DAG) const {
SDLoc DL(Op);
// If sign extension operations are disabled, allow sext_inreg only if operand
// is a vector extract. SIMD does not depend on sign extension operations, but
// allowing sext_inreg in this context lets us have simple patterns to select
// extract_lane_s instructions. Expanding sext_inreg everywhere would be
// simpler in this file, but would necessitate large and brittle patterns to
// undo the expansion and select extract_lane_s instructions.
assert(!Subtarget->hasSignExt() && Subtarget->hasSIMD128());
if (Op.getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
const SDValue &Extract = Op.getOperand(0);
MVT VecT = Extract.getOperand(0).getSimpleValueType();
MVT ExtractedLaneT = static_cast<VTSDNode *>(Op.getOperand(1).getNode())
->getVT()
.getSimpleVT();
MVT ExtractedVecT =
MVT::getVectorVT(ExtractedLaneT, 128 / ExtractedLaneT.getSizeInBits());
if (ExtractedVecT == VecT)
return Op;
// Bitcast vector to appropriate type to ensure ISel pattern coverage
const SDValue &Index = Extract.getOperand(1);
unsigned IndexVal =
static_cast<ConstantSDNode *>(Index.getNode())->getZExtValue();
unsigned Scale =
ExtractedVecT.getVectorNumElements() / VecT.getVectorNumElements();
assert(Scale > 1);
SDValue NewIndex =
DAG.getConstant(IndexVal * Scale, DL, Index.getValueType());
SDValue NewExtract = DAG.getNode(
ISD::EXTRACT_VECTOR_ELT, DL, Extract.getValueType(),
DAG.getBitcast(ExtractedVecT, Extract.getOperand(0)), NewIndex);
return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Op.getValueType(),
NewExtract, Op.getOperand(1));
}
// Otherwise expand
return SDValue();
}
SDValue WebAssemblyTargetLowering::LowerBUILD_VECTOR(SDValue Op,
SelectionDAG &DAG) const {
SDLoc DL(Op);
const EVT VecT = Op.getValueType();
const EVT LaneT = Op.getOperand(0).getValueType();
const size_t Lanes = Op.getNumOperands();
bool CanSwizzle = Subtarget->hasUnimplementedSIMD128() && VecT == MVT::v16i8;
// BUILD_VECTORs are lowered to the instruction that initializes the highest
// possible number of lanes at once followed by a sequence of replace_lane
// instructions to individually initialize any remaining lanes.
// TODO: Tune this. For example, lanewise swizzling is very expensive, so
// swizzled lanes should be given greater weight.
// TODO: Investigate building vectors by shuffling together vectors built by
// separately specialized means.
auto IsConstant = [](const SDValue &V) {
return V.getOpcode() == ISD::Constant || V.getOpcode() == ISD::ConstantFP;
};
// Returns the source vector and index vector pair if they exist. Checks for:
// (extract_vector_elt
// $src,
// (sign_extend_inreg (extract_vector_elt $indices, $i))
// )
auto GetSwizzleSrcs = [](size_t I, const SDValue &Lane) {
auto Bail = std::make_pair(SDValue(), SDValue());
if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
return Bail;
const SDValue &SwizzleSrc = Lane->getOperand(0);
const SDValue &IndexExt = Lane->getOperand(1);
if (IndexExt->getOpcode() != ISD::SIGN_EXTEND_INREG)
return Bail;
const SDValue &Index = IndexExt->getOperand(0);
if (Index->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
return Bail;
const SDValue &SwizzleIndices = Index->getOperand(0);
if (SwizzleSrc.getValueType() != MVT::v16i8 ||
SwizzleIndices.getValueType() != MVT::v16i8 ||
Index->getOperand(1)->getOpcode() != ISD::Constant ||
Index->getConstantOperandVal(1) != I)
return Bail;
return std::make_pair(SwizzleSrc, SwizzleIndices);
};
using ValueEntry = std::pair<SDValue, size_t>;
SmallVector<ValueEntry, 16> SplatValueCounts;
using SwizzleEntry = std::pair<std::pair<SDValue, SDValue>, size_t>;
SmallVector<SwizzleEntry, 16> SwizzleCounts;
auto AddCount = [](auto &Counts, const auto &Val) {
auto CountIt = std::find_if(Counts.begin(), Counts.end(),
[&Val](auto E) { return E.first == Val; });
if (CountIt == Counts.end()) {
Counts.emplace_back(Val, 1);
} else {
CountIt->second++;
}
};
auto GetMostCommon = [](auto &Counts) {
auto CommonIt =
std::max_element(Counts.begin(), Counts.end(),
[](auto A, auto B) { return A.second < B.second; });
assert(CommonIt != Counts.end() && "Unexpected all-undef build_vector");
return *CommonIt;
};
size_t NumConstantLanes = 0;
// Count eligible lanes for each type of vector creation op
for (size_t I = 0; I < Lanes; ++I) {
const SDValue &Lane = Op->getOperand(I);
if (Lane.isUndef())
continue;
AddCount(SplatValueCounts, Lane);
if (IsConstant(Lane)) {
NumConstantLanes++;
} else if (CanSwizzle) {
auto SwizzleSrcs = GetSwizzleSrcs(I, Lane);
if (SwizzleSrcs.first)
AddCount(SwizzleCounts, SwizzleSrcs);
}
}
SDValue SplatValue;
size_t NumSplatLanes;
std::tie(SplatValue, NumSplatLanes) = GetMostCommon(SplatValueCounts);
SDValue SwizzleSrc;
SDValue SwizzleIndices;
size_t NumSwizzleLanes = 0;
if (SwizzleCounts.size())
std::forward_as_tuple(std::tie(SwizzleSrc, SwizzleIndices),
NumSwizzleLanes) = GetMostCommon(SwizzleCounts);
// Predicate returning true if the lane is properly initialized by the
// original instruction
std::function<bool(size_t, const SDValue &)> IsLaneConstructed;
SDValue Result;
if (Subtarget->hasUnimplementedSIMD128()) {
// Prefer swizzles over vector consts over splats
if (NumSwizzleLanes >= NumSplatLanes &&
NumSwizzleLanes >= NumConstantLanes) {
Result = DAG.getNode(WebAssemblyISD::SWIZZLE, DL, VecT, SwizzleSrc,
SwizzleIndices);
auto Swizzled = std::make_pair(SwizzleSrc, SwizzleIndices);
IsLaneConstructed = [&, Swizzled](size_t I, const SDValue &Lane) {
return Swizzled == GetSwizzleSrcs(I, Lane);
};
} else if (NumConstantLanes >= NumSplatLanes) {
SmallVector<SDValue, 16> ConstLanes;
for (const SDValue &Lane : Op->op_values()) {
if (IsConstant(Lane)) {
ConstLanes.push_back(Lane);
} else if (LaneT.isFloatingPoint()) {
ConstLanes.push_back(DAG.getConstantFP(0, DL, LaneT));
} else {
ConstLanes.push_back(DAG.getConstant(0, DL, LaneT));
}
}
Result = DAG.getBuildVector(VecT, DL, ConstLanes);
IsLaneConstructed = [&](size_t _, const SDValue &Lane) {
return IsConstant(Lane);
};
}
}
if (!Result) {
// Use a splat, but possibly a load_splat
LoadSDNode *SplattedLoad;
if (Subtarget->hasUnimplementedSIMD128() &&
(SplattedLoad = dyn_cast<LoadSDNode>(SplatValue)) &&
SplattedLoad->getMemoryVT() == VecT.getVectorElementType()) {
Result = DAG.getMemIntrinsicNode(
WebAssemblyISD::LOAD_SPLAT, DL, DAG.getVTList(VecT),
{SplattedLoad->getChain(), SplattedLoad->getBasePtr(),
SplattedLoad->getOffset()},
SplattedLoad->getMemoryVT(), SplattedLoad->getMemOperand());
} else {
Result = DAG.getSplatBuildVector(VecT, DL, SplatValue);
}
IsLaneConstructed = [&](size_t _, const SDValue &Lane) {
return Lane == SplatValue;
};
}
// Add replace_lane instructions for any unhandled values
for (size_t I = 0; I < Lanes; ++I) {
const SDValue &Lane = Op->getOperand(I);
if (!Lane.isUndef() && !IsLaneConstructed(I, Lane))
Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VecT, Result, Lane,
DAG.getConstant(I, DL, MVT::i32));
}
return Result;
}
SDValue
WebAssemblyTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
SelectionDAG &DAG) const {
SDLoc DL(Op);
ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Op.getNode())->getMask();
MVT VecType = Op.getOperand(0).getSimpleValueType();
assert(VecType.is128BitVector() && "Unexpected shuffle vector type");
size_t LaneBytes = VecType.getVectorElementType().getSizeInBits() / 8;
// Space for two vector args and sixteen mask indices
SDValue Ops[18];
size_t OpIdx = 0;
Ops[OpIdx++] = Op.getOperand(0);
Ops[OpIdx++] = Op.getOperand(1);
// Expand mask indices to byte indices and materialize them as operands
for (int M : Mask) {
for (size_t J = 0; J < LaneBytes; ++J) {
// Lower undefs (represented by -1 in mask) to zero
uint64_t ByteIndex = M == -1 ? 0 : (uint64_t)M * LaneBytes + J;
Ops[OpIdx++] = DAG.getConstant(ByteIndex, DL, MVT::i32);
}
}
return DAG.getNode(WebAssemblyISD::SHUFFLE, DL, Op.getValueType(), Ops);
}
SDValue WebAssemblyTargetLowering::LowerSETCC(SDValue Op,
SelectionDAG &DAG) const {
SDLoc DL(Op);
// The legalizer does not know how to expand the comparison modes of i64x2
// vectors because no comparison modes are supported. We could solve this by
// expanding all i64x2 SETCC nodes, but that seems to expand f64x2 SETCC nodes
// (which return i64x2 results) as well. So instead we manually unroll i64x2
// comparisons here.
assert(Subtarget->hasUnimplementedSIMD128());
assert(Op->getOperand(0)->getSimpleValueType(0) == MVT::v2i64);
SmallVector<SDValue, 2> LHS, RHS;
DAG.ExtractVectorElements(Op->getOperand(0), LHS);
DAG.ExtractVectorElements(Op->getOperand(1), RHS);
const SDValue &CC = Op->getOperand(2);
auto MakeLane = [&](unsigned I) {
return DAG.getNode(ISD::SELECT_CC, DL, MVT::i64, LHS[I], RHS[I],
DAG.getConstant(uint64_t(-1), DL, MVT::i64),
DAG.getConstant(uint64_t(0), DL, MVT::i64), CC);
};
return DAG.getBuildVector(Op->getValueType(0), DL,
{MakeLane(0), MakeLane(1)});
}
SDValue
WebAssemblyTargetLowering::LowerAccessVectorElement(SDValue Op,
SelectionDAG &DAG) const {
// Allow constant lane indices, expand variable lane indices
SDNode *IdxNode = Op.getOperand(Op.getNumOperands() - 1).getNode();
if (isa<ConstantSDNode>(IdxNode) || IdxNode->isUndef())
return Op;
else
// Perform default expansion
return SDValue();
}
static SDValue unrollVectorShift(SDValue Op, SelectionDAG &DAG) {
EVT LaneT = Op.getSimpleValueType().getVectorElementType();
// 32-bit and 64-bit unrolled shifts will have proper semantics
if (LaneT.bitsGE(MVT::i32))
return DAG.UnrollVectorOp(Op.getNode());
// Otherwise mask the shift value to get proper semantics from 32-bit shift
SDLoc DL(Op);
SDValue ShiftVal = Op.getOperand(1);
uint64_t MaskVal = LaneT.getSizeInBits() - 1;
SDValue MaskedShiftVal = DAG.getNode(
ISD::AND, // mask opcode
DL, ShiftVal.getValueType(), // masked value type
ShiftVal, // original shift value operand
DAG.getConstant(MaskVal, DL, ShiftVal.getValueType()) // mask operand
);
return DAG.UnrollVectorOp(
DAG.getNode(Op.getOpcode(), // original shift opcode
DL, Op.getValueType(), // original return type
Op.getOperand(0), // original vector operand,
MaskedShiftVal // new masked shift value operand
)
.getNode());
}
SDValue WebAssemblyTargetLowering::LowerShift(SDValue Op,
SelectionDAG &DAG) const {
SDLoc DL(Op);
// Only manually lower vector shifts
assert(Op.getSimpleValueType().isVector());
// Unroll non-splat vector shifts
BuildVectorSDNode *ShiftVec;
SDValue SplatVal;
if (!(ShiftVec = dyn_cast<BuildVectorSDNode>(Op.getOperand(1).getNode())) ||
!(SplatVal = ShiftVec->getSplatValue()))
return unrollVectorShift(Op, DAG);
// All splats except i64x2 const splats are handled by patterns
auto *SplatConst = dyn_cast<ConstantSDNode>(SplatVal);
if (!SplatConst || Op.getSimpleValueType() != MVT::v2i64)
return Op;
// i64x2 const splats are custom lowered to avoid unnecessary wraps
unsigned Opcode;
switch (Op.getOpcode()) {
case ISD::SHL:
Opcode = WebAssemblyISD::VEC_SHL;
break;
case ISD::SRA:
Opcode = WebAssemblyISD::VEC_SHR_S;
break;
case ISD::SRL:
Opcode = WebAssemblyISD::VEC_SHR_U;
break;
default:
llvm_unreachable("unexpected opcode");
}
APInt Shift = SplatConst->getAPIntValue().zextOrTrunc(32);
return DAG.getNode(Opcode, DL, Op.getValueType(), Op.getOperand(0),
DAG.getConstant(Shift, DL, MVT::i32));
}
//===----------------------------------------------------------------------===//
// WebAssembly Optimization Hooks
//===----------------------------------------------------------------------===//