| //===- X86InstructionSelector.cpp -----------------------------------------===// | |
| // | |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |
| // See https://llvm.org/LICENSE.txt for license information. | |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| // | |
| //===----------------------------------------------------------------------===// | |
| /// \file | |
| /// This file implements the targeting of the InstructionSelector class for | |
| /// X86. | |
| /// \todo This should be generated by TableGen. | |
| //===----------------------------------------------------------------------===// | |
| #include "MCTargetDesc/X86BaseInfo.h" | |
| #include "X86.h" | |
| #include "X86InstrBuilder.h" | |
| #include "X86InstrInfo.h" | |
| #include "X86RegisterBankInfo.h" | |
| #include "X86RegisterInfo.h" | |
| #include "X86Subtarget.h" | |
| #include "X86TargetMachine.h" | |
| #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" | |
| #include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h" | |
| #include "llvm/CodeGen/GlobalISel/Utils.h" | |
| #include "llvm/CodeGen/MachineBasicBlock.h" | |
| #include "llvm/CodeGen/MachineConstantPool.h" | |
| #include "llvm/CodeGen/MachineFunction.h" | |
| #include "llvm/CodeGen/MachineInstr.h" | |
| #include "llvm/CodeGen/MachineInstrBuilder.h" | |
| #include "llvm/CodeGen/MachineMemOperand.h" | |
| #include "llvm/CodeGen/MachineOperand.h" | |
| #include "llvm/CodeGen/MachineRegisterInfo.h" | |
| #include "llvm/CodeGen/RegisterBank.h" | |
| #include "llvm/CodeGen/TargetOpcodes.h" | |
| #include "llvm/CodeGen/TargetRegisterInfo.h" | |
| #include "llvm/IR/DataLayout.h" | |
| #include "llvm/IR/InstrTypes.h" | |
| #include "llvm/IR/IntrinsicsX86.h" | |
| #include "llvm/Support/AtomicOrdering.h" | |
| #include "llvm/Support/CodeGen.h" | |
| #include "llvm/Support/Debug.h" | |
| #include "llvm/Support/ErrorHandling.h" | |
| #include "llvm/Support/LowLevelTypeImpl.h" | |
| #include "llvm/Support/MathExtras.h" | |
| #include "llvm/Support/raw_ostream.h" | |
| #include <cassert> | |
| #include <cstdint> | |
| #include <tuple> | |
| #define DEBUG_TYPE "X86-isel" | |
| using namespace llvm; | |
| namespace { | |
| #define GET_GLOBALISEL_PREDICATE_BITSET | |
| #include "X86GenGlobalISel.inc" | |
| #undef GET_GLOBALISEL_PREDICATE_BITSET | |
| class X86InstructionSelector : public InstructionSelector { | |
| public: | |
| X86InstructionSelector(const X86TargetMachine &TM, const X86Subtarget &STI, | |
| const X86RegisterBankInfo &RBI); | |
| bool select(MachineInstr &I) override; | |
| static const char *getName() { return DEBUG_TYPE; } | |
| private: | |
| /// tblgen-erated 'select' implementation, used as the initial selector for | |
| /// the patterns that don't require complex C++. | |
| bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; | |
| // TODO: remove after supported by Tablegen-erated instruction selection. | |
| unsigned getLoadStoreOp(const LLT &Ty, const RegisterBank &RB, unsigned Opc, | |
| Align Alignment) const; | |
| bool selectLoadStoreOp(MachineInstr &I, MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const; | |
| bool selectFrameIndexOrGep(MachineInstr &I, MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const; | |
| bool selectGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const; | |
| bool selectConstant(MachineInstr &I, MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const; | |
| bool selectTruncOrPtrToInt(MachineInstr &I, MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const; | |
| bool selectZext(MachineInstr &I, MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const; | |
| bool selectAnyext(MachineInstr &I, MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const; | |
| bool selectCmp(MachineInstr &I, MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const; | |
| bool selectFCmp(MachineInstr &I, MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const; | |
| bool selectUadde(MachineInstr &I, MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const; | |
| bool selectDebugInstr(MachineInstr &I, MachineRegisterInfo &MRI) const; | |
| bool selectCopy(MachineInstr &I, MachineRegisterInfo &MRI) const; | |
| bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI, | |
| MachineFunction &MF); | |
| bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI, | |
| MachineFunction &MF); | |
| bool selectInsert(MachineInstr &I, MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const; | |
| bool selectExtract(MachineInstr &I, MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const; | |
| bool selectCondBranch(MachineInstr &I, MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const; | |
| bool selectTurnIntoCOPY(MachineInstr &I, MachineRegisterInfo &MRI, | |
| const unsigned DstReg, | |
| const TargetRegisterClass *DstRC, | |
| const unsigned SrcReg, | |
| const TargetRegisterClass *SrcRC) const; | |
| bool materializeFP(MachineInstr &I, MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const; | |
| bool selectImplicitDefOrPHI(MachineInstr &I, MachineRegisterInfo &MRI) const; | |
| bool selectDivRem(MachineInstr &I, MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const; | |
| bool selectIntrinsicWSideEffects(MachineInstr &I, MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const; | |
| // emit insert subreg instruction and insert it before MachineInstr &I | |
| bool emitInsertSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I, | |
| MachineRegisterInfo &MRI, MachineFunction &MF) const; | |
| // emit extract subreg instruction and insert it before MachineInstr &I | |
| bool emitExtractSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I, | |
| MachineRegisterInfo &MRI, MachineFunction &MF) const; | |
| const TargetRegisterClass *getRegClass(LLT Ty, const RegisterBank &RB) const; | |
| const TargetRegisterClass *getRegClass(LLT Ty, unsigned Reg, | |
| MachineRegisterInfo &MRI) const; | |
| const X86TargetMachine &TM; | |
| const X86Subtarget &STI; | |
| const X86InstrInfo &TII; | |
| const X86RegisterInfo &TRI; | |
| const X86RegisterBankInfo &RBI; | |
| #define GET_GLOBALISEL_PREDICATES_DECL | |
| #include "X86GenGlobalISel.inc" | |
| #undef GET_GLOBALISEL_PREDICATES_DECL | |
| #define GET_GLOBALISEL_TEMPORARIES_DECL | |
| #include "X86GenGlobalISel.inc" | |
| #undef GET_GLOBALISEL_TEMPORARIES_DECL | |
| }; | |
| } // end anonymous namespace | |
| #define GET_GLOBALISEL_IMPL | |
| #include "X86GenGlobalISel.inc" | |
| #undef GET_GLOBALISEL_IMPL | |
| X86InstructionSelector::X86InstructionSelector(const X86TargetMachine &TM, | |
| const X86Subtarget &STI, | |
| const X86RegisterBankInfo &RBI) | |
| : TM(TM), STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), | |
| RBI(RBI), | |
| #define GET_GLOBALISEL_PREDICATES_INIT | |
| #include "X86GenGlobalISel.inc" | |
| #undef GET_GLOBALISEL_PREDICATES_INIT | |
| #define GET_GLOBALISEL_TEMPORARIES_INIT | |
| #include "X86GenGlobalISel.inc" | |
| #undef GET_GLOBALISEL_TEMPORARIES_INIT | |
| { | |
| } | |
| // FIXME: This should be target-independent, inferred from the types declared | |
| // for each class in the bank. | |
| const TargetRegisterClass * | |
| X86InstructionSelector::getRegClass(LLT Ty, const RegisterBank &RB) const { | |
| if (RB.getID() == X86::GPRRegBankID) { | |
| if (Ty.getSizeInBits() <= 8) | |
| return &X86::GR8RegClass; | |
| if (Ty.getSizeInBits() == 16) | |
| return &X86::GR16RegClass; | |
| if (Ty.getSizeInBits() == 32) | |
| return &X86::GR32RegClass; | |
| if (Ty.getSizeInBits() == 64) | |
| return &X86::GR64RegClass; | |
| } | |
| if (RB.getID() == X86::VECRRegBankID) { | |
| if (Ty.getSizeInBits() == 16) | |
| return STI.hasAVX512() ? &X86::FR16XRegClass : &X86::FR16RegClass; | |
| if (Ty.getSizeInBits() == 32) | |
| return STI.hasAVX512() ? &X86::FR32XRegClass : &X86::FR32RegClass; | |
| if (Ty.getSizeInBits() == 64) | |
| return STI.hasAVX512() ? &X86::FR64XRegClass : &X86::FR64RegClass; | |
| if (Ty.getSizeInBits() == 128) | |
| return STI.hasAVX512() ? &X86::VR128XRegClass : &X86::VR128RegClass; | |
| if (Ty.getSizeInBits() == 256) | |
| return STI.hasAVX512() ? &X86::VR256XRegClass : &X86::VR256RegClass; | |
| if (Ty.getSizeInBits() == 512) | |
| return &X86::VR512RegClass; | |
| } | |
| llvm_unreachable("Unknown RegBank!"); | |
| } | |
| const TargetRegisterClass * | |
| X86InstructionSelector::getRegClass(LLT Ty, unsigned Reg, | |
| MachineRegisterInfo &MRI) const { | |
| const RegisterBank &RegBank = *RBI.getRegBank(Reg, MRI, TRI); | |
| return getRegClass(Ty, RegBank); | |
| } | |
| static unsigned getSubRegIndex(const TargetRegisterClass *RC) { | |
| unsigned SubIdx = X86::NoSubRegister; | |
| if (RC == &X86::GR32RegClass) { | |
| SubIdx = X86::sub_32bit; | |
| } else if (RC == &X86::GR16RegClass) { | |
| SubIdx = X86::sub_16bit; | |
| } else if (RC == &X86::GR8RegClass) { | |
| SubIdx = X86::sub_8bit; | |
| } | |
| return SubIdx; | |
| } | |
| static const TargetRegisterClass *getRegClassFromGRPhysReg(Register Reg) { | |
| assert(Reg.isPhysical()); | |
| if (X86::GR64RegClass.contains(Reg)) | |
| return &X86::GR64RegClass; | |
| if (X86::GR32RegClass.contains(Reg)) | |
| return &X86::GR32RegClass; | |
| if (X86::GR16RegClass.contains(Reg)) | |
| return &X86::GR16RegClass; | |
| if (X86::GR8RegClass.contains(Reg)) | |
| return &X86::GR8RegClass; | |
| llvm_unreachable("Unknown RegClass for PhysReg!"); | |
| } | |
| // FIXME: We need some sort of API in RBI/TRI to allow generic code to | |
| // constrain operands of simple instructions given a TargetRegisterClass | |
| // and LLT | |
| bool X86InstructionSelector::selectDebugInstr(MachineInstr &I, | |
| MachineRegisterInfo &MRI) const { | |
| for (MachineOperand &MO : I.operands()) { | |
| if (!MO.isReg()) | |
| continue; | |
| Register Reg = MO.getReg(); | |
| if (!Reg) | |
| continue; | |
| if (Reg.isPhysical()) | |
| continue; | |
| LLT Ty = MRI.getType(Reg); | |
| const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg); | |
| const TargetRegisterClass *RC = | |
| RegClassOrBank.dyn_cast<const TargetRegisterClass *>(); | |
| if (!RC) { | |
| const RegisterBank &RB = *RegClassOrBank.get<const RegisterBank *>(); | |
| RC = getRegClass(Ty, RB); | |
| if (!RC) { | |
| LLVM_DEBUG( | |
| dbgs() << "Warning: DBG_VALUE operand has unexpected size/bank\n"); | |
| break; | |
| } | |
| } | |
| RBI.constrainGenericRegister(Reg, *RC, MRI); | |
| } | |
| return true; | |
| } | |
| // Set X86 Opcode and constrain DestReg. | |
| bool X86InstructionSelector::selectCopy(MachineInstr &I, | |
| MachineRegisterInfo &MRI) const { | |
| Register DstReg = I.getOperand(0).getReg(); | |
| const unsigned DstSize = RBI.getSizeInBits(DstReg, MRI, TRI); | |
| const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI); | |
| Register SrcReg = I.getOperand(1).getReg(); | |
| const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI); | |
| const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI); | |
| if (DstReg.isPhysical()) { | |
| assert(I.isCopy() && "Generic operators do not allow physical registers"); | |
| if (DstSize > SrcSize && SrcRegBank.getID() == X86::GPRRegBankID && | |
| DstRegBank.getID() == X86::GPRRegBankID) { | |
| const TargetRegisterClass *SrcRC = | |
| getRegClass(MRI.getType(SrcReg), SrcRegBank); | |
| const TargetRegisterClass *DstRC = getRegClassFromGRPhysReg(DstReg); | |
| if (SrcRC != DstRC) { | |
| // This case can be generated by ABI lowering, performe anyext | |
| Register ExtSrc = MRI.createVirtualRegister(DstRC); | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(TargetOpcode::SUBREG_TO_REG)) | |
| .addDef(ExtSrc) | |
| .addImm(0) | |
| .addReg(SrcReg) | |
| .addImm(getSubRegIndex(SrcRC)); | |
| I.getOperand(1).setReg(ExtSrc); | |
| } | |
| } | |
| return true; | |
| } | |
| assert((!SrcReg.isPhysical() || I.isCopy()) && | |
| "No phys reg on generic operators"); | |
| assert((DstSize == SrcSize || | |
| // Copies are a mean to setup initial types, the number of | |
| // bits may not exactly match. | |
| (SrcReg.isPhysical() && | |
| DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI))) && | |
| "Copy with different width?!"); | |
| const TargetRegisterClass *DstRC = | |
| getRegClass(MRI.getType(DstReg), DstRegBank); | |
| if (SrcRegBank.getID() == X86::GPRRegBankID && | |
| DstRegBank.getID() == X86::GPRRegBankID && SrcSize > DstSize && | |
| SrcReg.isPhysical()) { | |
| // Change the physical register to performe truncate. | |
| const TargetRegisterClass *SrcRC = getRegClassFromGRPhysReg(SrcReg); | |
| if (DstRC != SrcRC) { | |
| I.getOperand(1).setSubReg(getSubRegIndex(DstRC)); | |
| I.getOperand(1).substPhysReg(SrcReg, TRI); | |
| } | |
| } | |
| // No need to constrain SrcReg. It will get constrained when | |
| // we hit another of its use or its defs. | |
| // Copies do not have constraints. | |
| const TargetRegisterClass *OldRC = MRI.getRegClassOrNull(DstReg); | |
| if (!OldRC || !DstRC->hasSubClassEq(OldRC)) { | |
| if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { | |
| LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode()) | |
| << " operand\n"); | |
| return false; | |
| } | |
| } | |
| I.setDesc(TII.get(X86::COPY)); | |
| return true; | |
| } | |
| bool X86InstructionSelector::select(MachineInstr &I) { | |
| assert(I.getParent() && "Instruction should be in a basic block!"); | |
| assert(I.getParent()->getParent() && "Instruction should be in a function!"); | |
| MachineBasicBlock &MBB = *I.getParent(); | |
| MachineFunction &MF = *MBB.getParent(); | |
| MachineRegisterInfo &MRI = MF.getRegInfo(); | |
| unsigned Opcode = I.getOpcode(); | |
| if (!isPreISelGenericOpcode(Opcode)) { | |
| // Certain non-generic instructions also need some special handling. | |
| if (Opcode == TargetOpcode::LOAD_STACK_GUARD) | |
| return false; | |
| if (I.isCopy()) | |
| return selectCopy(I, MRI); | |
| if (I.isDebugInstr()) | |
| return selectDebugInstr(I, MRI); | |
| return true; | |
| } | |
| assert(I.getNumOperands() == I.getNumExplicitOperands() && | |
| "Generic instruction has unexpected implicit operands\n"); | |
| if (selectImpl(I, *CoverageInfo)) | |
| return true; | |
| LLVM_DEBUG(dbgs() << " C++ instruction selection: "; I.print(dbgs())); | |
| // TODO: This should be implemented by tblgen. | |
| switch (I.getOpcode()) { | |
| default: | |
| return false; | |
| case TargetOpcode::G_STORE: | |
| case TargetOpcode::G_LOAD: | |
| return selectLoadStoreOp(I, MRI, MF); | |
| case TargetOpcode::G_PTR_ADD: | |
| case TargetOpcode::G_FRAME_INDEX: | |
| return selectFrameIndexOrGep(I, MRI, MF); | |
| case TargetOpcode::G_GLOBAL_VALUE: | |
| return selectGlobalValue(I, MRI, MF); | |
| case TargetOpcode::G_CONSTANT: | |
| return selectConstant(I, MRI, MF); | |
| case TargetOpcode::G_FCONSTANT: | |
| return materializeFP(I, MRI, MF); | |
| case TargetOpcode::G_PTRTOINT: | |
| case TargetOpcode::G_TRUNC: | |
| return selectTruncOrPtrToInt(I, MRI, MF); | |
| case TargetOpcode::G_INTTOPTR: | |
| return selectCopy(I, MRI); | |
| case TargetOpcode::G_ZEXT: | |
| return selectZext(I, MRI, MF); | |
| case TargetOpcode::G_ANYEXT: | |
| return selectAnyext(I, MRI, MF); | |
| case TargetOpcode::G_ICMP: | |
| return selectCmp(I, MRI, MF); | |
| case TargetOpcode::G_FCMP: | |
| return selectFCmp(I, MRI, MF); | |
| case TargetOpcode::G_UADDE: | |
| return selectUadde(I, MRI, MF); | |
| case TargetOpcode::G_UNMERGE_VALUES: | |
| return selectUnmergeValues(I, MRI, MF); | |
| case TargetOpcode::G_MERGE_VALUES: | |
| case TargetOpcode::G_CONCAT_VECTORS: | |
| return selectMergeValues(I, MRI, MF); | |
| case TargetOpcode::G_EXTRACT: | |
| return selectExtract(I, MRI, MF); | |
| case TargetOpcode::G_INSERT: | |
| return selectInsert(I, MRI, MF); | |
| case TargetOpcode::G_BRCOND: | |
| return selectCondBranch(I, MRI, MF); | |
| case TargetOpcode::G_IMPLICIT_DEF: | |
| case TargetOpcode::G_PHI: | |
| return selectImplicitDefOrPHI(I, MRI); | |
| case TargetOpcode::G_SDIV: | |
| case TargetOpcode::G_UDIV: | |
| case TargetOpcode::G_SREM: | |
| case TargetOpcode::G_UREM: | |
| return selectDivRem(I, MRI, MF); | |
| case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: | |
| return selectIntrinsicWSideEffects(I, MRI, MF); | |
| } | |
| return false; | |
| } | |
| unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty, | |
| const RegisterBank &RB, | |
| unsigned Opc, | |
| Align Alignment) const { | |
| bool Isload = (Opc == TargetOpcode::G_LOAD); | |
| bool HasAVX = STI.hasAVX(); | |
| bool HasAVX512 = STI.hasAVX512(); | |
| bool HasVLX = STI.hasVLX(); | |
| if (Ty == LLT::scalar(8)) { | |
| if (X86::GPRRegBankID == RB.getID()) | |
| return Isload ? X86::MOV8rm : X86::MOV8mr; | |
| } else if (Ty == LLT::scalar(16)) { | |
| if (X86::GPRRegBankID == RB.getID()) | |
| return Isload ? X86::MOV16rm : X86::MOV16mr; | |
| } else if (Ty == LLT::scalar(32) || Ty == LLT::pointer(0, 32)) { | |
| if (X86::GPRRegBankID == RB.getID()) | |
| return Isload ? X86::MOV32rm : X86::MOV32mr; | |
| if (X86::VECRRegBankID == RB.getID()) | |
| return Isload ? (HasAVX512 ? X86::VMOVSSZrm_alt : | |
| HasAVX ? X86::VMOVSSrm_alt : | |
| X86::MOVSSrm_alt) | |
| : (HasAVX512 ? X86::VMOVSSZmr : | |
| HasAVX ? X86::VMOVSSmr : | |
| X86::MOVSSmr); | |
| } else if (Ty == LLT::scalar(64) || Ty == LLT::pointer(0, 64)) { | |
| if (X86::GPRRegBankID == RB.getID()) | |
| return Isload ? X86::MOV64rm : X86::MOV64mr; | |
| if (X86::VECRRegBankID == RB.getID()) | |
| return Isload ? (HasAVX512 ? X86::VMOVSDZrm_alt : | |
| HasAVX ? X86::VMOVSDrm_alt : | |
| X86::MOVSDrm_alt) | |
| : (HasAVX512 ? X86::VMOVSDZmr : | |
| HasAVX ? X86::VMOVSDmr : | |
| X86::MOVSDmr); | |
| } else if (Ty.isVector() && Ty.getSizeInBits() == 128) { | |
| if (Alignment >= Align(16)) | |
| return Isload ? (HasVLX ? X86::VMOVAPSZ128rm | |
| : HasAVX512 | |
| ? X86::VMOVAPSZ128rm_NOVLX | |
| : HasAVX ? X86::VMOVAPSrm : X86::MOVAPSrm) | |
| : (HasVLX ? X86::VMOVAPSZ128mr | |
| : HasAVX512 | |
| ? X86::VMOVAPSZ128mr_NOVLX | |
| : HasAVX ? X86::VMOVAPSmr : X86::MOVAPSmr); | |
| else | |
| return Isload ? (HasVLX ? X86::VMOVUPSZ128rm | |
| : HasAVX512 | |
| ? X86::VMOVUPSZ128rm_NOVLX | |
| : HasAVX ? X86::VMOVUPSrm : X86::MOVUPSrm) | |
| : (HasVLX ? X86::VMOVUPSZ128mr | |
| : HasAVX512 | |
| ? X86::VMOVUPSZ128mr_NOVLX | |
| : HasAVX ? X86::VMOVUPSmr : X86::MOVUPSmr); | |
| } else if (Ty.isVector() && Ty.getSizeInBits() == 256) { | |
| if (Alignment >= Align(32)) | |
| return Isload ? (HasVLX ? X86::VMOVAPSZ256rm | |
| : HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX | |
| : X86::VMOVAPSYrm) | |
| : (HasVLX ? X86::VMOVAPSZ256mr | |
| : HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX | |
| : X86::VMOVAPSYmr); | |
| else | |
| return Isload ? (HasVLX ? X86::VMOVUPSZ256rm | |
| : HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX | |
| : X86::VMOVUPSYrm) | |
| : (HasVLX ? X86::VMOVUPSZ256mr | |
| : HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX | |
| : X86::VMOVUPSYmr); | |
| } else if (Ty.isVector() && Ty.getSizeInBits() == 512) { | |
| if (Alignment >= Align(64)) | |
| return Isload ? X86::VMOVAPSZrm : X86::VMOVAPSZmr; | |
| else | |
| return Isload ? X86::VMOVUPSZrm : X86::VMOVUPSZmr; | |
| } | |
| return Opc; | |
| } | |
| // Fill in an address from the given instruction. | |
| static void X86SelectAddress(const MachineInstr &I, | |
| const MachineRegisterInfo &MRI, | |
| X86AddressMode &AM) { | |
| assert(I.getOperand(0).isReg() && "unsupported opperand."); | |
| assert(MRI.getType(I.getOperand(0).getReg()).isPointer() && | |
| "unsupported type."); | |
| if (I.getOpcode() == TargetOpcode::G_PTR_ADD) { | |
| if (auto COff = getIConstantVRegSExtVal(I.getOperand(2).getReg(), MRI)) { | |
| int64_t Imm = *COff; | |
| if (isInt<32>(Imm)) { // Check for displacement overflow. | |
| AM.Disp = static_cast<int32_t>(Imm); | |
| AM.Base.Reg = I.getOperand(1).getReg(); | |
| return; | |
| } | |
| } | |
| } else if (I.getOpcode() == TargetOpcode::G_FRAME_INDEX) { | |
| AM.Base.FrameIndex = I.getOperand(1).getIndex(); | |
| AM.BaseType = X86AddressMode::FrameIndexBase; | |
| return; | |
| } | |
| // Default behavior. | |
| AM.Base.Reg = I.getOperand(0).getReg(); | |
| } | |
| bool X86InstructionSelector::selectLoadStoreOp(MachineInstr &I, | |
| MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const { | |
| unsigned Opc = I.getOpcode(); | |
| assert((Opc == TargetOpcode::G_STORE || Opc == TargetOpcode::G_LOAD) && | |
| "unexpected instruction"); | |
| const Register DefReg = I.getOperand(0).getReg(); | |
| LLT Ty = MRI.getType(DefReg); | |
| const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI); | |
| assert(I.hasOneMemOperand()); | |
| auto &MemOp = **I.memoperands_begin(); | |
| if (MemOp.isAtomic()) { | |
| // Note: for unordered operations, we rely on the fact the appropriate MMO | |
| // is already on the instruction we're mutating, and thus we don't need to | |
| // make any changes. So long as we select an opcode which is capable of | |
| // loading or storing the appropriate size atomically, the rest of the | |
| // backend is required to respect the MMO state. | |
| if (!MemOp.isUnordered()) { | |
| LLVM_DEBUG(dbgs() << "Atomic ordering not supported yet\n"); | |
| return false; | |
| } | |
| if (MemOp.getAlign() < Ty.getSizeInBits() / 8) { | |
| LLVM_DEBUG(dbgs() << "Unaligned atomics not supported yet\n"); | |
| return false; | |
| } | |
| } | |
| unsigned NewOpc = getLoadStoreOp(Ty, RB, Opc, MemOp.getAlign()); | |
| if (NewOpc == Opc) | |
| return false; | |
| X86AddressMode AM; | |
| X86SelectAddress(*MRI.getVRegDef(I.getOperand(1).getReg()), MRI, AM); | |
| I.setDesc(TII.get(NewOpc)); | |
| MachineInstrBuilder MIB(MF, I); | |
| if (Opc == TargetOpcode::G_LOAD) { | |
| I.removeOperand(1); | |
| addFullAddress(MIB, AM); | |
| } else { | |
| // G_STORE (VAL, Addr), X86Store instruction (Addr, VAL) | |
| I.removeOperand(1); | |
| I.removeOperand(0); | |
| addFullAddress(MIB, AM).addUse(DefReg); | |
| } | |
| return constrainSelectedInstRegOperands(I, TII, TRI, RBI); | |
| } | |
| static unsigned getLeaOP(LLT Ty, const X86Subtarget &STI) { | |
| if (Ty == LLT::pointer(0, 64)) | |
| return X86::LEA64r; | |
| else if (Ty == LLT::pointer(0, 32)) | |
| return STI.isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r; | |
| else | |
| llvm_unreachable("Can't get LEA opcode. Unsupported type."); | |
| } | |
| bool X86InstructionSelector::selectFrameIndexOrGep(MachineInstr &I, | |
| MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const { | |
| unsigned Opc = I.getOpcode(); | |
| assert((Opc == TargetOpcode::G_FRAME_INDEX || Opc == TargetOpcode::G_PTR_ADD) && | |
| "unexpected instruction"); | |
| const Register DefReg = I.getOperand(0).getReg(); | |
| LLT Ty = MRI.getType(DefReg); | |
| // Use LEA to calculate frame index and GEP | |
| unsigned NewOpc = getLeaOP(Ty, STI); | |
| I.setDesc(TII.get(NewOpc)); | |
| MachineInstrBuilder MIB(MF, I); | |
| if (Opc == TargetOpcode::G_FRAME_INDEX) { | |
| addOffset(MIB, 0); | |
| } else { | |
| MachineOperand &InxOp = I.getOperand(2); | |
| I.addOperand(InxOp); // set IndexReg | |
| InxOp.ChangeToImmediate(1); // set Scale | |
| MIB.addImm(0).addReg(0); | |
| } | |
| return constrainSelectedInstRegOperands(I, TII, TRI, RBI); | |
| } | |
| bool X86InstructionSelector::selectGlobalValue(MachineInstr &I, | |
| MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const { | |
| assert((I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE) && | |
| "unexpected instruction"); | |
| auto GV = I.getOperand(1).getGlobal(); | |
| if (GV->isThreadLocal()) { | |
| return false; // TODO: we don't support TLS yet. | |
| } | |
| // Can't handle alternate code models yet. | |
| if (TM.getCodeModel() != CodeModel::Small) | |
| return false; | |
| X86AddressMode AM; | |
| AM.GV = GV; | |
| AM.GVOpFlags = STI.classifyGlobalReference(GV); | |
| // TODO: The ABI requires an extra load. not supported yet. | |
| if (isGlobalStubReference(AM.GVOpFlags)) | |
| return false; | |
| // TODO: This reference is relative to the pic base. not supported yet. | |
| if (isGlobalRelativeToPICBase(AM.GVOpFlags)) | |
| return false; | |
| if (STI.isPICStyleRIPRel()) { | |
| // Use rip-relative addressing. | |
| assert(AM.Base.Reg == 0 && AM.IndexReg == 0); | |
| AM.Base.Reg = X86::RIP; | |
| } | |
| const Register DefReg = I.getOperand(0).getReg(); | |
| LLT Ty = MRI.getType(DefReg); | |
| unsigned NewOpc = getLeaOP(Ty, STI); | |
| I.setDesc(TII.get(NewOpc)); | |
| MachineInstrBuilder MIB(MF, I); | |
| I.removeOperand(1); | |
| addFullAddress(MIB, AM); | |
| return constrainSelectedInstRegOperands(I, TII, TRI, RBI); | |
| } | |
| bool X86InstructionSelector::selectConstant(MachineInstr &I, | |
| MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const { | |
| assert((I.getOpcode() == TargetOpcode::G_CONSTANT) && | |
| "unexpected instruction"); | |
| const Register DefReg = I.getOperand(0).getReg(); | |
| LLT Ty = MRI.getType(DefReg); | |
| if (RBI.getRegBank(DefReg, MRI, TRI)->getID() != X86::GPRRegBankID) | |
| return false; | |
| uint64_t Val = 0; | |
| if (I.getOperand(1).isCImm()) { | |
| Val = I.getOperand(1).getCImm()->getZExtValue(); | |
| I.getOperand(1).ChangeToImmediate(Val); | |
| } else if (I.getOperand(1).isImm()) { | |
| Val = I.getOperand(1).getImm(); | |
| } else | |
| llvm_unreachable("Unsupported operand type."); | |
| unsigned NewOpc; | |
| switch (Ty.getSizeInBits()) { | |
| case 8: | |
| NewOpc = X86::MOV8ri; | |
| break; | |
| case 16: | |
| NewOpc = X86::MOV16ri; | |
| break; | |
| case 32: | |
| NewOpc = X86::MOV32ri; | |
| break; | |
| case 64: | |
| // TODO: in case isUInt<32>(Val), X86::MOV32ri can be used | |
| if (isInt<32>(Val)) | |
| NewOpc = X86::MOV64ri32; | |
| else | |
| NewOpc = X86::MOV64ri; | |
| break; | |
| default: | |
| llvm_unreachable("Can't select G_CONSTANT, unsupported type."); | |
| } | |
| I.setDesc(TII.get(NewOpc)); | |
| return constrainSelectedInstRegOperands(I, TII, TRI, RBI); | |
| } | |
| // Helper function for selectTruncOrPtrToInt and selectAnyext. | |
| // Returns true if DstRC lives on a floating register class and | |
| // SrcRC lives on a 128-bit vector class. | |
| static bool canTurnIntoCOPY(const TargetRegisterClass *DstRC, | |
| const TargetRegisterClass *SrcRC) { | |
| return (DstRC == &X86::FR32RegClass || DstRC == &X86::FR32XRegClass || | |
| DstRC == &X86::FR64RegClass || DstRC == &X86::FR64XRegClass) && | |
| (SrcRC == &X86::VR128RegClass || SrcRC == &X86::VR128XRegClass); | |
| } | |
| bool X86InstructionSelector::selectTurnIntoCOPY( | |
| MachineInstr &I, MachineRegisterInfo &MRI, const unsigned DstReg, | |
| const TargetRegisterClass *DstRC, const unsigned SrcReg, | |
| const TargetRegisterClass *SrcRC) const { | |
| if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) || | |
| !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { | |
| LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode()) | |
| << " operand\n"); | |
| return false; | |
| } | |
| I.setDesc(TII.get(X86::COPY)); | |
| return true; | |
| } | |
| bool X86InstructionSelector::selectTruncOrPtrToInt(MachineInstr &I, | |
| MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const { | |
| assert((I.getOpcode() == TargetOpcode::G_TRUNC || | |
| I.getOpcode() == TargetOpcode::G_PTRTOINT) && | |
| "unexpected instruction"); | |
| const Register DstReg = I.getOperand(0).getReg(); | |
| const Register SrcReg = I.getOperand(1).getReg(); | |
| const LLT DstTy = MRI.getType(DstReg); | |
| const LLT SrcTy = MRI.getType(SrcReg); | |
| const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI); | |
| const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI); | |
| if (DstRB.getID() != SrcRB.getID()) { | |
| LLVM_DEBUG(dbgs() << TII.getName(I.getOpcode()) | |
| << " input/output on different banks\n"); | |
| return false; | |
| } | |
| const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB); | |
| const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB); | |
| if (!DstRC || !SrcRC) | |
| return false; | |
| // If that's truncation of the value that lives on the vector class and goes | |
| // into the floating class, just replace it with copy, as we are able to | |
| // select it as a regular move. | |
| if (canTurnIntoCOPY(DstRC, SrcRC)) | |
| return selectTurnIntoCOPY(I, MRI, DstReg, DstRC, SrcReg, SrcRC); | |
| if (DstRB.getID() != X86::GPRRegBankID) | |
| return false; | |
| unsigned SubIdx; | |
| if (DstRC == SrcRC) { | |
| // Nothing to be done | |
| SubIdx = X86::NoSubRegister; | |
| } else if (DstRC == &X86::GR32RegClass) { | |
| SubIdx = X86::sub_32bit; | |
| } else if (DstRC == &X86::GR16RegClass) { | |
| SubIdx = X86::sub_16bit; | |
| } else if (DstRC == &X86::GR8RegClass) { | |
| SubIdx = X86::sub_8bit; | |
| } else { | |
| return false; | |
| } | |
| SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx); | |
| if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) || | |
| !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { | |
| LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode()) | |
| << "\n"); | |
| return false; | |
| } | |
| I.getOperand(1).setSubReg(SubIdx); | |
| I.setDesc(TII.get(X86::COPY)); | |
| return true; | |
| } | |
| bool X86InstructionSelector::selectZext(MachineInstr &I, | |
| MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const { | |
| assert((I.getOpcode() == TargetOpcode::G_ZEXT) && "unexpected instruction"); | |
| const Register DstReg = I.getOperand(0).getReg(); | |
| const Register SrcReg = I.getOperand(1).getReg(); | |
| const LLT DstTy = MRI.getType(DstReg); | |
| const LLT SrcTy = MRI.getType(SrcReg); | |
| assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(16)) && | |
| "8=>16 Zext is handled by tablegen"); | |
| assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(32)) && | |
| "8=>32 Zext is handled by tablegen"); | |
| assert(!(SrcTy == LLT::scalar(16) && DstTy == LLT::scalar(32)) && | |
| "16=>32 Zext is handled by tablegen"); | |
| assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(64)) && | |
| "8=>64 Zext is handled by tablegen"); | |
| assert(!(SrcTy == LLT::scalar(16) && DstTy == LLT::scalar(64)) && | |
| "16=>64 Zext is handled by tablegen"); | |
| assert(!(SrcTy == LLT::scalar(32) && DstTy == LLT::scalar(64)) && | |
| "32=>64 Zext is handled by tablegen"); | |
| if (SrcTy != LLT::scalar(1)) | |
| return false; | |
| unsigned AndOpc; | |
| if (DstTy == LLT::scalar(8)) | |
| AndOpc = X86::AND8ri; | |
| else if (DstTy == LLT::scalar(16)) | |
| AndOpc = X86::AND16ri8; | |
| else if (DstTy == LLT::scalar(32)) | |
| AndOpc = X86::AND32ri8; | |
| else if (DstTy == LLT::scalar(64)) | |
| AndOpc = X86::AND64ri8; | |
| else | |
| return false; | |
| Register DefReg = SrcReg; | |
| if (DstTy != LLT::scalar(8)) { | |
| Register ImpDefReg = | |
| MRI.createVirtualRegister(getRegClass(DstTy, DstReg, MRI)); | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(TargetOpcode::IMPLICIT_DEF), ImpDefReg); | |
| DefReg = MRI.createVirtualRegister(getRegClass(DstTy, DstReg, MRI)); | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(TargetOpcode::INSERT_SUBREG), DefReg) | |
| .addReg(ImpDefReg) | |
| .addReg(SrcReg) | |
| .addImm(X86::sub_8bit); | |
| } | |
| MachineInstr &AndInst = | |
| *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AndOpc), DstReg) | |
| .addReg(DefReg) | |
| .addImm(1); | |
| constrainSelectedInstRegOperands(AndInst, TII, TRI, RBI); | |
| I.eraseFromParent(); | |
| return true; | |
| } | |
| bool X86InstructionSelector::selectAnyext(MachineInstr &I, | |
| MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const { | |
| assert((I.getOpcode() == TargetOpcode::G_ANYEXT) && "unexpected instruction"); | |
| const Register DstReg = I.getOperand(0).getReg(); | |
| const Register SrcReg = I.getOperand(1).getReg(); | |
| const LLT DstTy = MRI.getType(DstReg); | |
| const LLT SrcTy = MRI.getType(SrcReg); | |
| const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI); | |
| const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI); | |
| assert(DstRB.getID() == SrcRB.getID() && | |
| "G_ANYEXT input/output on different banks\n"); | |
| assert(DstTy.getSizeInBits() > SrcTy.getSizeInBits() && | |
| "G_ANYEXT incorrect operand size"); | |
| const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB); | |
| const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB); | |
| // If that's ANY_EXT of the value that lives on the floating class and goes | |
| // into the vector class, just replace it with copy, as we are able to select | |
| // it as a regular move. | |
| if (canTurnIntoCOPY(SrcRC, DstRC)) | |
| return selectTurnIntoCOPY(I, MRI, SrcReg, SrcRC, DstReg, DstRC); | |
| if (DstRB.getID() != X86::GPRRegBankID) | |
| return false; | |
| if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) || | |
| !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { | |
| LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode()) | |
| << " operand\n"); | |
| return false; | |
| } | |
| if (SrcRC == DstRC) { | |
| I.setDesc(TII.get(X86::COPY)); | |
| return true; | |
| } | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(TargetOpcode::SUBREG_TO_REG)) | |
| .addDef(DstReg) | |
| .addImm(0) | |
| .addReg(SrcReg) | |
| .addImm(getSubRegIndex(SrcRC)); | |
| I.eraseFromParent(); | |
| return true; | |
| } | |
| bool X86InstructionSelector::selectCmp(MachineInstr &I, | |
| MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const { | |
| assert((I.getOpcode() == TargetOpcode::G_ICMP) && "unexpected instruction"); | |
| X86::CondCode CC; | |
| bool SwapArgs; | |
| std::tie(CC, SwapArgs) = X86::getX86ConditionCode( | |
| (CmpInst::Predicate)I.getOperand(1).getPredicate()); | |
| Register LHS = I.getOperand(2).getReg(); | |
| Register RHS = I.getOperand(3).getReg(); | |
| if (SwapArgs) | |
| std::swap(LHS, RHS); | |
| unsigned OpCmp; | |
| LLT Ty = MRI.getType(LHS); | |
| switch (Ty.getSizeInBits()) { | |
| default: | |
| return false; | |
| case 8: | |
| OpCmp = X86::CMP8rr; | |
| break; | |
| case 16: | |
| OpCmp = X86::CMP16rr; | |
| break; | |
| case 32: | |
| OpCmp = X86::CMP32rr; | |
| break; | |
| case 64: | |
| OpCmp = X86::CMP64rr; | |
| break; | |
| } | |
| MachineInstr &CmpInst = | |
| *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp)) | |
| .addReg(LHS) | |
| .addReg(RHS); | |
| MachineInstr &SetInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(X86::SETCCr), I.getOperand(0).getReg()).addImm(CC); | |
| constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI); | |
| constrainSelectedInstRegOperands(SetInst, TII, TRI, RBI); | |
| I.eraseFromParent(); | |
| return true; | |
| } | |
| bool X86InstructionSelector::selectFCmp(MachineInstr &I, | |
| MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const { | |
| assert((I.getOpcode() == TargetOpcode::G_FCMP) && "unexpected instruction"); | |
| Register LhsReg = I.getOperand(2).getReg(); | |
| Register RhsReg = I.getOperand(3).getReg(); | |
| CmpInst::Predicate Predicate = | |
| (CmpInst::Predicate)I.getOperand(1).getPredicate(); | |
| // FCMP_OEQ and FCMP_UNE cannot be checked with a single instruction. | |
| static const uint16_t SETFOpcTable[2][3] = { | |
| {X86::COND_E, X86::COND_NP, X86::AND8rr}, | |
| {X86::COND_NE, X86::COND_P, X86::OR8rr}}; | |
| const uint16_t *SETFOpc = nullptr; | |
| switch (Predicate) { | |
| default: | |
| break; | |
| case CmpInst::FCMP_OEQ: | |
| SETFOpc = &SETFOpcTable[0][0]; | |
| break; | |
| case CmpInst::FCMP_UNE: | |
| SETFOpc = &SETFOpcTable[1][0]; | |
| break; | |
| } | |
| // Compute the opcode for the CMP instruction. | |
| unsigned OpCmp; | |
| LLT Ty = MRI.getType(LhsReg); | |
| switch (Ty.getSizeInBits()) { | |
| default: | |
| return false; | |
| case 32: | |
| OpCmp = X86::UCOMISSrr; | |
| break; | |
| case 64: | |
| OpCmp = X86::UCOMISDrr; | |
| break; | |
| } | |
| Register ResultReg = I.getOperand(0).getReg(); | |
| RBI.constrainGenericRegister( | |
| ResultReg, | |
| *getRegClass(LLT::scalar(8), *RBI.getRegBank(ResultReg, MRI, TRI)), MRI); | |
| if (SETFOpc) { | |
| MachineInstr &CmpInst = | |
| *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp)) | |
| .addReg(LhsReg) | |
| .addReg(RhsReg); | |
| Register FlagReg1 = MRI.createVirtualRegister(&X86::GR8RegClass); | |
| Register FlagReg2 = MRI.createVirtualRegister(&X86::GR8RegClass); | |
| MachineInstr &Set1 = *BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(X86::SETCCr), FlagReg1).addImm(SETFOpc[0]); | |
| MachineInstr &Set2 = *BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(X86::SETCCr), FlagReg2).addImm(SETFOpc[1]); | |
| MachineInstr &Set3 = *BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(SETFOpc[2]), ResultReg) | |
| .addReg(FlagReg1) | |
| .addReg(FlagReg2); | |
| constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI); | |
| constrainSelectedInstRegOperands(Set1, TII, TRI, RBI); | |
| constrainSelectedInstRegOperands(Set2, TII, TRI, RBI); | |
| constrainSelectedInstRegOperands(Set3, TII, TRI, RBI); | |
| I.eraseFromParent(); | |
| return true; | |
| } | |
| X86::CondCode CC; | |
| bool SwapArgs; | |
| std::tie(CC, SwapArgs) = X86::getX86ConditionCode(Predicate); | |
| assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code."); | |
| if (SwapArgs) | |
| std::swap(LhsReg, RhsReg); | |
| // Emit a compare of LHS/RHS. | |
| MachineInstr &CmpInst = | |
| *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp)) | |
| .addReg(LhsReg) | |
| .addReg(RhsReg); | |
| MachineInstr &Set = | |
| *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::SETCCr), ResultReg).addImm(CC); | |
| constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI); | |
| constrainSelectedInstRegOperands(Set, TII, TRI, RBI); | |
| I.eraseFromParent(); | |
| return true; | |
| } | |
| bool X86InstructionSelector::selectUadde(MachineInstr &I, | |
| MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const { | |
| assert((I.getOpcode() == TargetOpcode::G_UADDE) && "unexpected instruction"); | |
| const Register DstReg = I.getOperand(0).getReg(); | |
| const Register CarryOutReg = I.getOperand(1).getReg(); | |
| const Register Op0Reg = I.getOperand(2).getReg(); | |
| const Register Op1Reg = I.getOperand(3).getReg(); | |
| Register CarryInReg = I.getOperand(4).getReg(); | |
| const LLT DstTy = MRI.getType(DstReg); | |
| if (DstTy != LLT::scalar(32)) | |
| return false; | |
| // find CarryIn def instruction. | |
| MachineInstr *Def = MRI.getVRegDef(CarryInReg); | |
| while (Def->getOpcode() == TargetOpcode::G_TRUNC) { | |
| CarryInReg = Def->getOperand(1).getReg(); | |
| Def = MRI.getVRegDef(CarryInReg); | |
| } | |
| unsigned Opcode; | |
| if (Def->getOpcode() == TargetOpcode::G_UADDE) { | |
| // carry set by prev ADD. | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), X86::EFLAGS) | |
| .addReg(CarryInReg); | |
| if (!RBI.constrainGenericRegister(CarryInReg, X86::GR32RegClass, MRI)) | |
| return false; | |
| Opcode = X86::ADC32rr; | |
| } else if (auto val = getIConstantVRegVal(CarryInReg, MRI)) { | |
| // carry is constant, support only 0. | |
| if (*val != 0) | |
| return false; | |
| Opcode = X86::ADD32rr; | |
| } else | |
| return false; | |
| MachineInstr &AddInst = | |
| *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode), DstReg) | |
| .addReg(Op0Reg) | |
| .addReg(Op1Reg); | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), CarryOutReg) | |
| .addReg(X86::EFLAGS); | |
| if (!constrainSelectedInstRegOperands(AddInst, TII, TRI, RBI) || | |
| !RBI.constrainGenericRegister(CarryOutReg, X86::GR32RegClass, MRI)) | |
| return false; | |
| I.eraseFromParent(); | |
| return true; | |
| } | |
| bool X86InstructionSelector::selectExtract(MachineInstr &I, | |
| MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const { | |
| assert((I.getOpcode() == TargetOpcode::G_EXTRACT) && | |
| "unexpected instruction"); | |
| const Register DstReg = I.getOperand(0).getReg(); | |
| const Register SrcReg = I.getOperand(1).getReg(); | |
| int64_t Index = I.getOperand(2).getImm(); | |
| const LLT DstTy = MRI.getType(DstReg); | |
| const LLT SrcTy = MRI.getType(SrcReg); | |
| // Meanwile handle vector type only. | |
| if (!DstTy.isVector()) | |
| return false; | |
| if (Index % DstTy.getSizeInBits() != 0) | |
| return false; // Not extract subvector. | |
| if (Index == 0) { | |
| // Replace by extract subreg copy. | |
| if (!emitExtractSubreg(DstReg, SrcReg, I, MRI, MF)) | |
| return false; | |
| I.eraseFromParent(); | |
| return true; | |
| } | |
| bool HasAVX = STI.hasAVX(); | |
| bool HasAVX512 = STI.hasAVX512(); | |
| bool HasVLX = STI.hasVLX(); | |
| if (SrcTy.getSizeInBits() == 256 && DstTy.getSizeInBits() == 128) { | |
| if (HasVLX) | |
| I.setDesc(TII.get(X86::VEXTRACTF32x4Z256rr)); | |
| else if (HasAVX) | |
| I.setDesc(TII.get(X86::VEXTRACTF128rr)); | |
| else | |
| return false; | |
| } else if (SrcTy.getSizeInBits() == 512 && HasAVX512) { | |
| if (DstTy.getSizeInBits() == 128) | |
| I.setDesc(TII.get(X86::VEXTRACTF32x4Zrr)); | |
| else if (DstTy.getSizeInBits() == 256) | |
| I.setDesc(TII.get(X86::VEXTRACTF64x4Zrr)); | |
| else | |
| return false; | |
| } else | |
| return false; | |
| // Convert to X86 VEXTRACT immediate. | |
| Index = Index / DstTy.getSizeInBits(); | |
| I.getOperand(2).setImm(Index); | |
| return constrainSelectedInstRegOperands(I, TII, TRI, RBI); | |
| } | |
| bool X86InstructionSelector::emitExtractSubreg(unsigned DstReg, unsigned SrcReg, | |
| MachineInstr &I, | |
| MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const { | |
| const LLT DstTy = MRI.getType(DstReg); | |
| const LLT SrcTy = MRI.getType(SrcReg); | |
| unsigned SubIdx = X86::NoSubRegister; | |
| if (!DstTy.isVector() || !SrcTy.isVector()) | |
| return false; | |
| assert(SrcTy.getSizeInBits() > DstTy.getSizeInBits() && | |
| "Incorrect Src/Dst register size"); | |
| if (DstTy.getSizeInBits() == 128) | |
| SubIdx = X86::sub_xmm; | |
| else if (DstTy.getSizeInBits() == 256) | |
| SubIdx = X86::sub_ymm; | |
| else | |
| return false; | |
| const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI); | |
| const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI); | |
| SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx); | |
| if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) || | |
| !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { | |
| LLVM_DEBUG(dbgs() << "Failed to constrain EXTRACT_SUBREG\n"); | |
| return false; | |
| } | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), DstReg) | |
| .addReg(SrcReg, 0, SubIdx); | |
| return true; | |
| } | |
| bool X86InstructionSelector::emitInsertSubreg(unsigned DstReg, unsigned SrcReg, | |
| MachineInstr &I, | |
| MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const { | |
| const LLT DstTy = MRI.getType(DstReg); | |
| const LLT SrcTy = MRI.getType(SrcReg); | |
| unsigned SubIdx = X86::NoSubRegister; | |
| // TODO: support scalar types | |
| if (!DstTy.isVector() || !SrcTy.isVector()) | |
| return false; | |
| assert(SrcTy.getSizeInBits() < DstTy.getSizeInBits() && | |
| "Incorrect Src/Dst register size"); | |
| if (SrcTy.getSizeInBits() == 128) | |
| SubIdx = X86::sub_xmm; | |
| else if (SrcTy.getSizeInBits() == 256) | |
| SubIdx = X86::sub_ymm; | |
| else | |
| return false; | |
| const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI); | |
| const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI); | |
| if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) || | |
| !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { | |
| LLVM_DEBUG(dbgs() << "Failed to constrain INSERT_SUBREG\n"); | |
| return false; | |
| } | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY)) | |
| .addReg(DstReg, RegState::DefineNoRead, SubIdx) | |
| .addReg(SrcReg); | |
| return true; | |
| } | |
| bool X86InstructionSelector::selectInsert(MachineInstr &I, | |
| MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const { | |
| assert((I.getOpcode() == TargetOpcode::G_INSERT) && "unexpected instruction"); | |
| const Register DstReg = I.getOperand(0).getReg(); | |
| const Register SrcReg = I.getOperand(1).getReg(); | |
| const Register InsertReg = I.getOperand(2).getReg(); | |
| int64_t Index = I.getOperand(3).getImm(); | |
| const LLT DstTy = MRI.getType(DstReg); | |
| const LLT InsertRegTy = MRI.getType(InsertReg); | |
| // Meanwile handle vector type only. | |
| if (!DstTy.isVector()) | |
| return false; | |
| if (Index % InsertRegTy.getSizeInBits() != 0) | |
| return false; // Not insert subvector. | |
| if (Index == 0 && MRI.getVRegDef(SrcReg)->isImplicitDef()) { | |
| // Replace by subreg copy. | |
| if (!emitInsertSubreg(DstReg, InsertReg, I, MRI, MF)) | |
| return false; | |
| I.eraseFromParent(); | |
| return true; | |
| } | |
| bool HasAVX = STI.hasAVX(); | |
| bool HasAVX512 = STI.hasAVX512(); | |
| bool HasVLX = STI.hasVLX(); | |
| if (DstTy.getSizeInBits() == 256 && InsertRegTy.getSizeInBits() == 128) { | |
| if (HasVLX) | |
| I.setDesc(TII.get(X86::VINSERTF32x4Z256rr)); | |
| else if (HasAVX) | |
| I.setDesc(TII.get(X86::VINSERTF128rr)); | |
| else | |
| return false; | |
| } else if (DstTy.getSizeInBits() == 512 && HasAVX512) { | |
| if (InsertRegTy.getSizeInBits() == 128) | |
| I.setDesc(TII.get(X86::VINSERTF32x4Zrr)); | |
| else if (InsertRegTy.getSizeInBits() == 256) | |
| I.setDesc(TII.get(X86::VINSERTF64x4Zrr)); | |
| else | |
| return false; | |
| } else | |
| return false; | |
| // Convert to X86 VINSERT immediate. | |
| Index = Index / InsertRegTy.getSizeInBits(); | |
| I.getOperand(3).setImm(Index); | |
| return constrainSelectedInstRegOperands(I, TII, TRI, RBI); | |
| } | |
| bool X86InstructionSelector::selectUnmergeValues( | |
| MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) { | |
| assert((I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES) && | |
| "unexpected instruction"); | |
| // Split to extracts. | |
| unsigned NumDefs = I.getNumOperands() - 1; | |
| Register SrcReg = I.getOperand(NumDefs).getReg(); | |
| unsigned DefSize = MRI.getType(I.getOperand(0).getReg()).getSizeInBits(); | |
| for (unsigned Idx = 0; Idx < NumDefs; ++Idx) { | |
| MachineInstr &ExtrInst = | |
| *BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(TargetOpcode::G_EXTRACT), I.getOperand(Idx).getReg()) | |
| .addReg(SrcReg) | |
| .addImm(Idx * DefSize); | |
| if (!select(ExtrInst)) | |
| return false; | |
| } | |
| I.eraseFromParent(); | |
| return true; | |
| } | |
| bool X86InstructionSelector::selectMergeValues( | |
| MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) { | |
| assert((I.getOpcode() == TargetOpcode::G_MERGE_VALUES || | |
| I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS) && | |
| "unexpected instruction"); | |
| // Split to inserts. | |
| Register DstReg = I.getOperand(0).getReg(); | |
| Register SrcReg0 = I.getOperand(1).getReg(); | |
| const LLT DstTy = MRI.getType(DstReg); | |
| const LLT SrcTy = MRI.getType(SrcReg0); | |
| unsigned SrcSize = SrcTy.getSizeInBits(); | |
| const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI); | |
| // For the first src use insertSubReg. | |
| Register DefReg = MRI.createGenericVirtualRegister(DstTy); | |
| MRI.setRegBank(DefReg, RegBank); | |
| if (!emitInsertSubreg(DefReg, I.getOperand(1).getReg(), I, MRI, MF)) | |
| return false; | |
| for (unsigned Idx = 2; Idx < I.getNumOperands(); ++Idx) { | |
| Register Tmp = MRI.createGenericVirtualRegister(DstTy); | |
| MRI.setRegBank(Tmp, RegBank); | |
| MachineInstr &InsertInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(TargetOpcode::G_INSERT), Tmp) | |
| .addReg(DefReg) | |
| .addReg(I.getOperand(Idx).getReg()) | |
| .addImm((Idx - 1) * SrcSize); | |
| DefReg = Tmp; | |
| if (!select(InsertInst)) | |
| return false; | |
| } | |
| MachineInstr &CopyInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(TargetOpcode::COPY), DstReg) | |
| .addReg(DefReg); | |
| if (!select(CopyInst)) | |
| return false; | |
| I.eraseFromParent(); | |
| return true; | |
| } | |
| bool X86InstructionSelector::selectCondBranch(MachineInstr &I, | |
| MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const { | |
| assert((I.getOpcode() == TargetOpcode::G_BRCOND) && "unexpected instruction"); | |
| const Register CondReg = I.getOperand(0).getReg(); | |
| MachineBasicBlock *DestMBB = I.getOperand(1).getMBB(); | |
| MachineInstr &TestInst = | |
| *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::TEST8ri)) | |
| .addReg(CondReg) | |
| .addImm(1); | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::JCC_1)) | |
| .addMBB(DestMBB).addImm(X86::COND_NE); | |
| constrainSelectedInstRegOperands(TestInst, TII, TRI, RBI); | |
| I.eraseFromParent(); | |
| return true; | |
| } | |
| bool X86InstructionSelector::materializeFP(MachineInstr &I, | |
| MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const { | |
| assert((I.getOpcode() == TargetOpcode::G_FCONSTANT) && | |
| "unexpected instruction"); | |
| // Can't handle alternate code models yet. | |
| CodeModel::Model CM = TM.getCodeModel(); | |
| if (CM != CodeModel::Small && CM != CodeModel::Large) | |
| return false; | |
| const Register DstReg = I.getOperand(0).getReg(); | |
| const LLT DstTy = MRI.getType(DstReg); | |
| const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI); | |
| Align Alignment = Align(DstTy.getSizeInBytes()); | |
| const DebugLoc &DbgLoc = I.getDebugLoc(); | |
| unsigned Opc = | |
| getLoadStoreOp(DstTy, RegBank, TargetOpcode::G_LOAD, Alignment); | |
| // Create the load from the constant pool. | |
| const ConstantFP *CFP = I.getOperand(1).getFPImm(); | |
| unsigned CPI = MF.getConstantPool()->getConstantPoolIndex(CFP, Alignment); | |
| MachineInstr *LoadInst = nullptr; | |
| unsigned char OpFlag = STI.classifyLocalReference(nullptr); | |
| if (CM == CodeModel::Large && STI.is64Bit()) { | |
| // Under X86-64 non-small code model, GV (and friends) are 64-bits, so | |
| // they cannot be folded into immediate fields. | |
| Register AddrReg = MRI.createVirtualRegister(&X86::GR64RegClass); | |
| BuildMI(*I.getParent(), I, DbgLoc, TII.get(X86::MOV64ri), AddrReg) | |
| .addConstantPoolIndex(CPI, 0, OpFlag); | |
| MachineMemOperand *MMO = MF.getMachineMemOperand( | |
| MachinePointerInfo::getConstantPool(MF), MachineMemOperand::MOLoad, | |
| LLT::pointer(0, MF.getDataLayout().getPointerSizeInBits()), Alignment); | |
| LoadInst = | |
| addDirectMem(BuildMI(*I.getParent(), I, DbgLoc, TII.get(Opc), DstReg), | |
| AddrReg) | |
| .addMemOperand(MMO); | |
| } else if (CM == CodeModel::Small || !STI.is64Bit()) { | |
| // Handle the case when globals fit in our immediate field. | |
| // This is true for X86-32 always and X86-64 when in -mcmodel=small mode. | |
| // x86-32 PIC requires a PIC base register for constant pools. | |
| unsigned PICBase = 0; | |
| if (OpFlag == X86II::MO_PIC_BASE_OFFSET || OpFlag == X86II::MO_GOTOFF) { | |
| // PICBase can be allocated by TII.getGlobalBaseReg(&MF). | |
| // In DAGISEL the code that initialize it generated by the CGBR pass. | |
| return false; // TODO support the mode. | |
| } else if (STI.is64Bit() && TM.getCodeModel() == CodeModel::Small) | |
| PICBase = X86::RIP; | |
| LoadInst = addConstantPoolReference( | |
| BuildMI(*I.getParent(), I, DbgLoc, TII.get(Opc), DstReg), CPI, PICBase, | |
| OpFlag); | |
| } else | |
| return false; | |
| constrainSelectedInstRegOperands(*LoadInst, TII, TRI, RBI); | |
| I.eraseFromParent(); | |
| return true; | |
| } | |
| bool X86InstructionSelector::selectImplicitDefOrPHI( | |
| MachineInstr &I, MachineRegisterInfo &MRI) const { | |
| assert((I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF || | |
| I.getOpcode() == TargetOpcode::G_PHI) && | |
| "unexpected instruction"); | |
| Register DstReg = I.getOperand(0).getReg(); | |
| if (!MRI.getRegClassOrNull(DstReg)) { | |
| const LLT DstTy = MRI.getType(DstReg); | |
| const TargetRegisterClass *RC = getRegClass(DstTy, DstReg, MRI); | |
| if (!RBI.constrainGenericRegister(DstReg, *RC, MRI)) { | |
| LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode()) | |
| << " operand\n"); | |
| return false; | |
| } | |
| } | |
| if (I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF) | |
| I.setDesc(TII.get(X86::IMPLICIT_DEF)); | |
| else | |
| I.setDesc(TII.get(X86::PHI)); | |
| return true; | |
| } | |
| bool X86InstructionSelector::selectDivRem(MachineInstr &I, | |
| MachineRegisterInfo &MRI, | |
| MachineFunction &MF) const { | |
| // The implementation of this function is taken from X86FastISel. | |
| assert((I.getOpcode() == TargetOpcode::G_SDIV || | |
| I.getOpcode() == TargetOpcode::G_SREM || | |
| I.getOpcode() == TargetOpcode::G_UDIV || | |
| I.getOpcode() == TargetOpcode::G_UREM) && | |
| "unexpected instruction"); | |
| const Register DstReg = I.getOperand(0).getReg(); | |
| const Register Op1Reg = I.getOperand(1).getReg(); | |
| const Register Op2Reg = I.getOperand(2).getReg(); | |
| const LLT RegTy = MRI.getType(DstReg); | |
| assert(RegTy == MRI.getType(Op1Reg) && RegTy == MRI.getType(Op2Reg) && | |
| "Arguments and return value types must match"); | |
| const RegisterBank *RegRB = RBI.getRegBank(DstReg, MRI, TRI); | |
| if (!RegRB || RegRB->getID() != X86::GPRRegBankID) | |
| return false; | |
| const static unsigned NumTypes = 4; // i8, i16, i32, i64 | |
| const static unsigned NumOps = 4; // SDiv, SRem, UDiv, URem | |
| const static bool S = true; // IsSigned | |
| const static bool U = false; // !IsSigned | |
| const static unsigned Copy = TargetOpcode::COPY; | |
| // For the X86 IDIV instruction, in most cases the dividend | |
| // (numerator) must be in a specific register pair highreg:lowreg, | |
| // producing the quotient in lowreg and the remainder in highreg. | |
| // For most data types, to set up the instruction, the dividend is | |
| // copied into lowreg, and lowreg is sign-extended into highreg. The | |
| // exception is i8, where the dividend is defined as a single register rather | |
| // than a register pair, and we therefore directly sign-extend the dividend | |
| // into lowreg, instead of copying, and ignore the highreg. | |
| const static struct DivRemEntry { | |
| // The following portion depends only on the data type. | |
| unsigned SizeInBits; | |
| unsigned LowInReg; // low part of the register pair | |
| unsigned HighInReg; // high part of the register pair | |
| // The following portion depends on both the data type and the operation. | |
| struct DivRemResult { | |
| unsigned OpDivRem; // The specific DIV/IDIV opcode to use. | |
| unsigned OpSignExtend; // Opcode for sign-extending lowreg into | |
| // highreg, or copying a zero into highreg. | |
| unsigned OpCopy; // Opcode for copying dividend into lowreg, or | |
| // zero/sign-extending into lowreg for i8. | |
| unsigned DivRemResultReg; // Register containing the desired result. | |
| bool IsOpSigned; // Whether to use signed or unsigned form. | |
| } ResultTable[NumOps]; | |
| } OpTable[NumTypes] = { | |
| {8, | |
| X86::AX, | |
| 0, | |
| { | |
| {X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AL, S}, // SDiv | |
| {X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AH, S}, // SRem | |
| {X86::DIV8r, 0, X86::MOVZX16rr8, X86::AL, U}, // UDiv | |
| {X86::DIV8r, 0, X86::MOVZX16rr8, X86::AH, U}, // URem | |
| }}, // i8 | |
| {16, | |
| X86::AX, | |
| X86::DX, | |
| { | |
| {X86::IDIV16r, X86::CWD, Copy, X86::AX, S}, // SDiv | |
| {X86::IDIV16r, X86::CWD, Copy, X86::DX, S}, // SRem | |
| {X86::DIV16r, X86::MOV32r0, Copy, X86::AX, U}, // UDiv | |
| {X86::DIV16r, X86::MOV32r0, Copy, X86::DX, U}, // URem | |
| }}, // i16 | |
| {32, | |
| X86::EAX, | |
| X86::EDX, | |
| { | |
| {X86::IDIV32r, X86::CDQ, Copy, X86::EAX, S}, // SDiv | |
| {X86::IDIV32r, X86::CDQ, Copy, X86::EDX, S}, // SRem | |
| {X86::DIV32r, X86::MOV32r0, Copy, X86::EAX, U}, // UDiv | |
| {X86::DIV32r, X86::MOV32r0, Copy, X86::EDX, U}, // URem | |
| }}, // i32 | |
| {64, | |
| X86::RAX, | |
| X86::RDX, | |
| { | |
| {X86::IDIV64r, X86::CQO, Copy, X86::RAX, S}, // SDiv | |
| {X86::IDIV64r, X86::CQO, Copy, X86::RDX, S}, // SRem | |
| {X86::DIV64r, X86::MOV32r0, Copy, X86::RAX, U}, // UDiv | |
| {X86::DIV64r, X86::MOV32r0, Copy, X86::RDX, U}, // URem | |
| }}, // i64 | |
| }; | |
| auto OpEntryIt = llvm::find_if(OpTable, [RegTy](const DivRemEntry &El) { | |
| return El.SizeInBits == RegTy.getSizeInBits(); | |
| }); | |
| if (OpEntryIt == std::end(OpTable)) | |
| return false; | |
| unsigned OpIndex; | |
| switch (I.getOpcode()) { | |
| default: | |
| llvm_unreachable("Unexpected div/rem opcode"); | |
| case TargetOpcode::G_SDIV: | |
| OpIndex = 0; | |
| break; | |
| case TargetOpcode::G_SREM: | |
| OpIndex = 1; | |
| break; | |
| case TargetOpcode::G_UDIV: | |
| OpIndex = 2; | |
| break; | |
| case TargetOpcode::G_UREM: | |
| OpIndex = 3; | |
| break; | |
| } | |
| const DivRemEntry &TypeEntry = *OpEntryIt; | |
| const DivRemEntry::DivRemResult &OpEntry = TypeEntry.ResultTable[OpIndex]; | |
| const TargetRegisterClass *RegRC = getRegClass(RegTy, *RegRB); | |
| if (!RBI.constrainGenericRegister(Op1Reg, *RegRC, MRI) || | |
| !RBI.constrainGenericRegister(Op2Reg, *RegRC, MRI) || | |
| !RBI.constrainGenericRegister(DstReg, *RegRC, MRI)) { | |
| LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode()) | |
| << " operand\n"); | |
| return false; | |
| } | |
| // Move op1 into low-order input register. | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpEntry.OpCopy), | |
| TypeEntry.LowInReg) | |
| .addReg(Op1Reg); | |
| // Zero-extend or sign-extend into high-order input register. | |
| if (OpEntry.OpSignExtend) { | |
| if (OpEntry.IsOpSigned) | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(OpEntry.OpSignExtend)); | |
| else { | |
| Register Zero32 = MRI.createVirtualRegister(&X86::GR32RegClass); | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::MOV32r0), | |
| Zero32); | |
| // Copy the zero into the appropriate sub/super/identical physical | |
| // register. Unfortunately the operations needed are not uniform enough | |
| // to fit neatly into the table above. | |
| if (RegTy.getSizeInBits() == 16) { | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy), | |
| TypeEntry.HighInReg) | |
| .addReg(Zero32, 0, X86::sub_16bit); | |
| } else if (RegTy.getSizeInBits() == 32) { | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy), | |
| TypeEntry.HighInReg) | |
| .addReg(Zero32); | |
| } else if (RegTy.getSizeInBits() == 64) { | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(TargetOpcode::SUBREG_TO_REG), TypeEntry.HighInReg) | |
| .addImm(0) | |
| .addReg(Zero32) | |
| .addImm(X86::sub_32bit); | |
| } | |
| } | |
| } | |
| // Generate the DIV/IDIV instruction. | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpEntry.OpDivRem)) | |
| .addReg(Op2Reg); | |
| // For i8 remainder, we can't reference ah directly, as we'll end | |
| // up with bogus copies like %r9b = COPY %ah. Reference ax | |
| // instead to prevent ah references in a rex instruction. | |
| // | |
| // The current assumption of the fast register allocator is that isel | |
| // won't generate explicit references to the GR8_NOREX registers. If | |
| // the allocator and/or the backend get enhanced to be more robust in | |
| // that regard, this can be, and should be, removed. | |
| if ((I.getOpcode() == Instruction::SRem || | |
| I.getOpcode() == Instruction::URem) && | |
| OpEntry.DivRemResultReg == X86::AH && STI.is64Bit()) { | |
| Register SourceSuperReg = MRI.createVirtualRegister(&X86::GR16RegClass); | |
| Register ResultSuperReg = MRI.createVirtualRegister(&X86::GR16RegClass); | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy), SourceSuperReg) | |
| .addReg(X86::AX); | |
| // Shift AX right by 8 bits instead of using AH. | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::SHR16ri), | |
| ResultSuperReg) | |
| .addReg(SourceSuperReg) | |
| .addImm(8); | |
| // Now reference the 8-bit subreg of the result. | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(TargetOpcode::SUBREG_TO_REG)) | |
| .addDef(DstReg) | |
| .addImm(0) | |
| .addReg(ResultSuperReg) | |
| .addImm(X86::sub_8bit); | |
| } else { | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), | |
| DstReg) | |
| .addReg(OpEntry.DivRemResultReg); | |
| } | |
| I.eraseFromParent(); | |
| return true; | |
| } | |
| bool X86InstructionSelector::selectIntrinsicWSideEffects( | |
| MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) const { | |
| assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS && | |
| "unexpected instruction"); | |
| if (I.getOperand(0).getIntrinsicID() != Intrinsic::trap) | |
| return false; | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::TRAP)); | |
| I.eraseFromParent(); | |
| return true; | |
| } | |
| InstructionSelector * | |
| llvm::createX86InstructionSelector(const X86TargetMachine &TM, | |
| X86Subtarget &Subtarget, | |
| X86RegisterBankInfo &RBI) { | |
| return new X86InstructionSelector(TM, Subtarget, RBI); | |
| } |