| //===- AArch64InstructionSelector.cpp ----------------------------*- C++ -*-==// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| /// \file |
| /// This file implements the targeting of the InstructionSelector class for |
| /// AArch64. |
| /// \todo This should be generated by TableGen. |
| //===----------------------------------------------------------------------===// |
| |
| #include "AArch64InstrInfo.h" |
| #include "AArch64MachineFunctionInfo.h" |
| #include "AArch64RegisterBankInfo.h" |
| #include "AArch64RegisterInfo.h" |
| #include "AArch64Subtarget.h" |
| #include "AArch64TargetMachine.h" |
| #include "MCTargetDesc/AArch64AddressingModes.h" |
| #include "llvm/ADT/Optional.h" |
| #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" |
| #include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h" |
| #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" |
| #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" |
| #include "llvm/CodeGen/GlobalISel/Utils.h" |
| #include "llvm/CodeGen/MachineBasicBlock.h" |
| #include "llvm/CodeGen/MachineConstantPool.h" |
| #include "llvm/CodeGen/MachineFunction.h" |
| #include "llvm/CodeGen/MachineInstr.h" |
| #include "llvm/CodeGen/MachineInstrBuilder.h" |
| #include "llvm/CodeGen/MachineOperand.h" |
| #include "llvm/CodeGen/MachineRegisterInfo.h" |
| #include "llvm/IR/Type.h" |
| #include "llvm/IR/IntrinsicsAArch64.h" |
| #include "llvm/Support/Debug.h" |
| #include "llvm/Support/raw_ostream.h" |
| |
| #define DEBUG_TYPE "aarch64-isel" |
| |
| using namespace llvm; |
| |
| namespace { |
| |
| #define GET_GLOBALISEL_PREDICATE_BITSET |
| #include "AArch64GenGlobalISel.inc" |
| #undef GET_GLOBALISEL_PREDICATE_BITSET |
| |
| class AArch64InstructionSelector : public InstructionSelector { |
| public: |
| AArch64InstructionSelector(const AArch64TargetMachine &TM, |
| const AArch64Subtarget &STI, |
| const AArch64RegisterBankInfo &RBI); |
| |
| bool select(MachineInstr &I) override; |
| static const char *getName() { return DEBUG_TYPE; } |
| |
| void setupMF(MachineFunction &MF, GISelKnownBits &KB, |
| CodeGenCoverage &CoverageInfo) override { |
| InstructionSelector::setupMF(MF, KB, CoverageInfo); |
| |
| // hasFnAttribute() is expensive to call on every BRCOND selection, so |
| // cache it here for each run of the selector. |
| ProduceNonFlagSettingCondBr = |
| !MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening); |
| } |
| |
| private: |
| /// tblgen-erated 'select' implementation, used as the initial selector for |
| /// the patterns that don't require complex C++. |
| bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; |
| |
| // A lowering phase that runs before any selection attempts. |
| |
| void preISelLower(MachineInstr &I) const; |
| |
| // An early selection function that runs before the selectImpl() call. |
| bool earlySelect(MachineInstr &I) const; |
| |
| bool earlySelectSHL(MachineInstr &I, MachineRegisterInfo &MRI) const; |
| |
| /// Eliminate same-sized cross-bank copies into stores before selectImpl(). |
| void contractCrossBankCopyIntoStore(MachineInstr &I, |
| MachineRegisterInfo &MRI) const; |
| |
| bool selectVaStartAAPCS(MachineInstr &I, MachineFunction &MF, |
| MachineRegisterInfo &MRI) const; |
| bool selectVaStartDarwin(MachineInstr &I, MachineFunction &MF, |
| MachineRegisterInfo &MRI) const; |
| |
| bool selectCompareBranch(MachineInstr &I, MachineFunction &MF, |
| MachineRegisterInfo &MRI) const; |
| |
| bool selectVectorASHR(MachineInstr &I, MachineRegisterInfo &MRI) const; |
| bool selectVectorSHL(MachineInstr &I, MachineRegisterInfo &MRI) const; |
| |
| // Helper to generate an equivalent of scalar_to_vector into a new register, |
| // returned via 'Dst'. |
| MachineInstr *emitScalarToVector(unsigned EltSize, |
| const TargetRegisterClass *DstRC, |
| Register Scalar, |
| MachineIRBuilder &MIRBuilder) const; |
| |
| /// Emit a lane insert into \p DstReg, or a new vector register if None is |
| /// provided. |
| /// |
| /// The lane inserted into is defined by \p LaneIdx. The vector source |
| /// register is given by \p SrcReg. The register containing the element is |
| /// given by \p EltReg. |
| MachineInstr *emitLaneInsert(Optional<Register> DstReg, Register SrcReg, |
| Register EltReg, unsigned LaneIdx, |
| const RegisterBank &RB, |
| MachineIRBuilder &MIRBuilder) const; |
| bool selectInsertElt(MachineInstr &I, MachineRegisterInfo &MRI) const; |
| bool selectBuildVector(MachineInstr &I, MachineRegisterInfo &MRI) const; |
| bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI) const; |
| bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI) const; |
| |
| bool selectShuffleVector(MachineInstr &I, MachineRegisterInfo &MRI) const; |
| bool selectExtractElt(MachineInstr &I, MachineRegisterInfo &MRI) const; |
| bool selectConcatVectors(MachineInstr &I, MachineRegisterInfo &MRI) const; |
| bool selectSplitVectorUnmerge(MachineInstr &I, |
| MachineRegisterInfo &MRI) const; |
| bool selectIntrinsicWithSideEffects(MachineInstr &I, |
| MachineRegisterInfo &MRI) const; |
| bool selectIntrinsic(MachineInstr &I, MachineRegisterInfo &MRI) const; |
| bool selectVectorICmp(MachineInstr &I, MachineRegisterInfo &MRI) const; |
| bool selectIntrinsicTrunc(MachineInstr &I, MachineRegisterInfo &MRI) const; |
| bool selectIntrinsicRound(MachineInstr &I, MachineRegisterInfo &MRI) const; |
| bool selectJumpTable(MachineInstr &I, MachineRegisterInfo &MRI) const; |
| bool selectBrJT(MachineInstr &I, MachineRegisterInfo &MRI) const; |
| bool selectTLSGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI) const; |
| |
| unsigned emitConstantPoolEntry(Constant *CPVal, MachineFunction &MF) const; |
| MachineInstr *emitLoadFromConstantPool(Constant *CPVal, |
| MachineIRBuilder &MIRBuilder) const; |
| |
| // Emit a vector concat operation. |
| MachineInstr *emitVectorConcat(Optional<Register> Dst, Register Op1, |
| Register Op2, |
| MachineIRBuilder &MIRBuilder) const; |
| MachineInstr *emitIntegerCompare(MachineOperand &LHS, MachineOperand &RHS, |
| MachineOperand &Predicate, |
| MachineIRBuilder &MIRBuilder) const; |
| MachineInstr *emitADD(Register DefReg, MachineOperand &LHS, MachineOperand &RHS, |
| MachineIRBuilder &MIRBuilder) const; |
| MachineInstr *emitCMN(MachineOperand &LHS, MachineOperand &RHS, |
| MachineIRBuilder &MIRBuilder) const; |
| MachineInstr *emitTST(const Register &LHS, const Register &RHS, |
| MachineIRBuilder &MIRBuilder) const; |
| MachineInstr *emitExtractVectorElt(Optional<Register> DstReg, |
| const RegisterBank &DstRB, LLT ScalarTy, |
| Register VecReg, unsigned LaneIdx, |
| MachineIRBuilder &MIRBuilder) const; |
| |
| /// Helper function for selecting G_FCONSTANT. If the G_FCONSTANT can be |
| /// materialized using a FMOV instruction, then update MI and return it. |
| /// Otherwise, do nothing and return a nullptr. |
| MachineInstr *emitFMovForFConstant(MachineInstr &MI, |
| MachineRegisterInfo &MRI) const; |
| |
| /// Emit a CSet for a compare. |
| MachineInstr *emitCSetForICMP(Register DefReg, unsigned Pred, |
| MachineIRBuilder &MIRBuilder) const; |
| |
| // Equivalent to the i32shift_a and friends from AArch64InstrInfo.td. |
| // We use these manually instead of using the importer since it doesn't |
| // support SDNodeXForm. |
| ComplexRendererFns selectShiftA_32(const MachineOperand &Root) const; |
| ComplexRendererFns selectShiftB_32(const MachineOperand &Root) const; |
| ComplexRendererFns selectShiftA_64(const MachineOperand &Root) const; |
| ComplexRendererFns selectShiftB_64(const MachineOperand &Root) const; |
| |
| ComplexRendererFns select12BitValueWithLeftShift(uint64_t Immed) const; |
| ComplexRendererFns selectArithImmed(MachineOperand &Root) const; |
| ComplexRendererFns selectNegArithImmed(MachineOperand &Root) const; |
| |
| ComplexRendererFns selectAddrModeUnscaled(MachineOperand &Root, |
| unsigned Size) const; |
| |
| ComplexRendererFns selectAddrModeUnscaled8(MachineOperand &Root) const { |
| return selectAddrModeUnscaled(Root, 1); |
| } |
| ComplexRendererFns selectAddrModeUnscaled16(MachineOperand &Root) const { |
| return selectAddrModeUnscaled(Root, 2); |
| } |
| ComplexRendererFns selectAddrModeUnscaled32(MachineOperand &Root) const { |
| return selectAddrModeUnscaled(Root, 4); |
| } |
| ComplexRendererFns selectAddrModeUnscaled64(MachineOperand &Root) const { |
| return selectAddrModeUnscaled(Root, 8); |
| } |
| ComplexRendererFns selectAddrModeUnscaled128(MachineOperand &Root) const { |
| return selectAddrModeUnscaled(Root, 16); |
| } |
| |
| ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root, |
| unsigned Size) const; |
| template <int Width> |
| ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root) const { |
| return selectAddrModeIndexed(Root, Width / 8); |
| } |
| |
| bool isWorthFoldingIntoExtendedReg(MachineInstr &MI, |
| const MachineRegisterInfo &MRI) const; |
| ComplexRendererFns |
| selectAddrModeShiftedExtendXReg(MachineOperand &Root, |
| unsigned SizeInBytes) const; |
| |
| /// Returns a \p ComplexRendererFns which contains a base, offset, and whether |
| /// or not a shift + extend should be folded into an addressing mode. Returns |
| /// None when this is not profitable or possible. |
| ComplexRendererFns |
| selectExtendedSHL(MachineOperand &Root, MachineOperand &Base, |
| MachineOperand &Offset, unsigned SizeInBytes, |
| bool WantsExt) const; |
| ComplexRendererFns selectAddrModeRegisterOffset(MachineOperand &Root) const; |
| ComplexRendererFns selectAddrModeXRO(MachineOperand &Root, |
| unsigned SizeInBytes) const; |
| template <int Width> |
| ComplexRendererFns selectAddrModeXRO(MachineOperand &Root) const { |
| return selectAddrModeXRO(Root, Width / 8); |
| } |
| |
| ComplexRendererFns selectAddrModeWRO(MachineOperand &Root, |
| unsigned SizeInBytes) const; |
| template <int Width> |
| ComplexRendererFns selectAddrModeWRO(MachineOperand &Root) const { |
| return selectAddrModeWRO(Root, Width / 8); |
| } |
| |
| ComplexRendererFns selectShiftedRegister(MachineOperand &Root) const; |
| |
| ComplexRendererFns selectArithShiftedRegister(MachineOperand &Root) const { |
| return selectShiftedRegister(Root); |
| } |
| |
| ComplexRendererFns selectLogicalShiftedRegister(MachineOperand &Root) const { |
| // TODO: selectShiftedRegister should allow for rotates on logical shifts. |
| // For now, make them the same. The only difference between the two is that |
| // logical shifts are allowed to fold in rotates. Otherwise, these are |
| // functionally the same. |
| return selectShiftedRegister(Root); |
| } |
| |
| /// Given an extend instruction, determine the correct shift-extend type for |
| /// that instruction. |
| /// |
| /// If the instruction is going to be used in a load or store, pass |
| /// \p IsLoadStore = true. |
| AArch64_AM::ShiftExtendType |
| getExtendTypeForInst(MachineInstr &MI, MachineRegisterInfo &MRI, |
| bool IsLoadStore = false) const; |
| |
| /// Instructions that accept extend modifiers like UXTW expect the register |
| /// being extended to be a GPR32. Narrow ExtReg to a 32-bit register using a |
| /// subregister copy if necessary. Return either ExtReg, or the result of the |
| /// new copy. |
| Register narrowExtendRegIfNeeded(Register ExtReg, |
| MachineIRBuilder &MIB) const; |
| ComplexRendererFns selectArithExtendedRegister(MachineOperand &Root) const; |
| |
| void renderTruncImm(MachineInstrBuilder &MIB, const MachineInstr &MI, |
| int OpIdx = -1) const; |
| void renderLogicalImm32(MachineInstrBuilder &MIB, const MachineInstr &I, |
| int OpIdx = -1) const; |
| void renderLogicalImm64(MachineInstrBuilder &MIB, const MachineInstr &I, |
| int OpIdx = -1) const; |
| |
| // Materialize a GlobalValue or BlockAddress using a movz+movk sequence. |
| void materializeLargeCMVal(MachineInstr &I, const Value *V, |
| unsigned OpFlags) const; |
| |
| // Optimization methods. |
| bool tryOptVectorShuffle(MachineInstr &I) const; |
| bool tryOptVectorDup(MachineInstr &MI) const; |
| bool tryOptSelect(MachineInstr &MI) const; |
| MachineInstr *tryFoldIntegerCompare(MachineOperand &LHS, MachineOperand &RHS, |
| MachineOperand &Predicate, |
| MachineIRBuilder &MIRBuilder) const; |
| |
| /// Return true if \p MI is a load or store of \p NumBytes bytes. |
| bool isLoadStoreOfNumBytes(const MachineInstr &MI, unsigned NumBytes) const; |
| |
| /// Returns true if \p MI is guaranteed to have the high-half of a 64-bit |
| /// register zeroed out. In other words, the result of MI has been explicitly |
| /// zero extended. |
| bool isDef32(const MachineInstr &MI) const; |
| |
| const AArch64TargetMachine &TM; |
| const AArch64Subtarget &STI; |
| const AArch64InstrInfo &TII; |
| const AArch64RegisterInfo &TRI; |
| const AArch64RegisterBankInfo &RBI; |
| |
| bool ProduceNonFlagSettingCondBr = false; |
| |
| #define GET_GLOBALISEL_PREDICATES_DECL |
| #include "AArch64GenGlobalISel.inc" |
| #undef GET_GLOBALISEL_PREDICATES_DECL |
| |
| // We declare the temporaries used by selectImpl() in the class to minimize the |
| // cost of constructing placeholder values. |
| #define GET_GLOBALISEL_TEMPORARIES_DECL |
| #include "AArch64GenGlobalISel.inc" |
| #undef GET_GLOBALISEL_TEMPORARIES_DECL |
| }; |
| |
| } // end anonymous namespace |
| |
| #define GET_GLOBALISEL_IMPL |
| #include "AArch64GenGlobalISel.inc" |
| #undef GET_GLOBALISEL_IMPL |
| |
| AArch64InstructionSelector::AArch64InstructionSelector( |
| const AArch64TargetMachine &TM, const AArch64Subtarget &STI, |
| const AArch64RegisterBankInfo &RBI) |
| : InstructionSelector(), TM(TM), STI(STI), TII(*STI.getInstrInfo()), |
| TRI(*STI.getRegisterInfo()), RBI(RBI), |
| #define GET_GLOBALISEL_PREDICATES_INIT |
| #include "AArch64GenGlobalISel.inc" |
| #undef GET_GLOBALISEL_PREDICATES_INIT |
| #define GET_GLOBALISEL_TEMPORARIES_INIT |
| #include "AArch64GenGlobalISel.inc" |
| #undef GET_GLOBALISEL_TEMPORARIES_INIT |
| { |
| } |
| |
| // FIXME: This should be target-independent, inferred from the types declared |
| // for each class in the bank. |
| static const TargetRegisterClass * |
| getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB, |
| const RegisterBankInfo &RBI, |
| bool GetAllRegSet = false) { |
| if (RB.getID() == AArch64::GPRRegBankID) { |
| if (Ty.getSizeInBits() <= 32) |
| return GetAllRegSet ? &AArch64::GPR32allRegClass |
| : &AArch64::GPR32RegClass; |
| if (Ty.getSizeInBits() == 64) |
| return GetAllRegSet ? &AArch64::GPR64allRegClass |
| : &AArch64::GPR64RegClass; |
| return nullptr; |
| } |
| |
| if (RB.getID() == AArch64::FPRRegBankID) { |
| if (Ty.getSizeInBits() <= 16) |
| return &AArch64::FPR16RegClass; |
| if (Ty.getSizeInBits() == 32) |
| return &AArch64::FPR32RegClass; |
| if (Ty.getSizeInBits() == 64) |
| return &AArch64::FPR64RegClass; |
| if (Ty.getSizeInBits() == 128) |
| return &AArch64::FPR128RegClass; |
| return nullptr; |
| } |
| |
| return nullptr; |
| } |
| |
| /// Given a register bank, and size in bits, return the smallest register class |
| /// that can represent that combination. |
| static const TargetRegisterClass * |
| getMinClassForRegBank(const RegisterBank &RB, unsigned SizeInBits, |
| bool GetAllRegSet = false) { |
| unsigned RegBankID = RB.getID(); |
| |
| if (RegBankID == AArch64::GPRRegBankID) { |
| if (SizeInBits <= 32) |
| return GetAllRegSet ? &AArch64::GPR32allRegClass |
| : &AArch64::GPR32RegClass; |
| if (SizeInBits == 64) |
| return GetAllRegSet ? &AArch64::GPR64allRegClass |
| : &AArch64::GPR64RegClass; |
| } |
| |
| if (RegBankID == AArch64::FPRRegBankID) { |
| switch (SizeInBits) { |
| default: |
| return nullptr; |
| case 8: |
| return &AArch64::FPR8RegClass; |
| case 16: |
| return &AArch64::FPR16RegClass; |
| case 32: |
| return &AArch64::FPR32RegClass; |
| case 64: |
| return &AArch64::FPR64RegClass; |
| case 128: |
| return &AArch64::FPR128RegClass; |
| } |
| } |
| |
| return nullptr; |
| } |
| |
| /// Returns the correct subregister to use for a given register class. |
| static bool getSubRegForClass(const TargetRegisterClass *RC, |
| const TargetRegisterInfo &TRI, unsigned &SubReg) { |
| switch (TRI.getRegSizeInBits(*RC)) { |
| case 8: |
| SubReg = AArch64::bsub; |
| break; |
| case 16: |
| SubReg = AArch64::hsub; |
| break; |
| case 32: |
| if (RC != &AArch64::FPR32RegClass) |
| SubReg = AArch64::sub_32; |
| else |
| SubReg = AArch64::ssub; |
| break; |
| case 64: |
| SubReg = AArch64::dsub; |
| break; |
| default: |
| LLVM_DEBUG( |
| dbgs() << "Couldn't find appropriate subregister for register class."); |
| return false; |
| } |
| |
| return true; |
| } |
| |
| /// Check whether \p I is a currently unsupported binary operation: |
| /// - it has an unsized type |
| /// - an operand is not a vreg |
| /// - all operands are not in the same bank |
| /// These are checks that should someday live in the verifier, but right now, |
| /// these are mostly limitations of the aarch64 selector. |
| static bool unsupportedBinOp(const MachineInstr &I, |
| const AArch64RegisterBankInfo &RBI, |
| const MachineRegisterInfo &MRI, |
| const AArch64RegisterInfo &TRI) { |
| LLT Ty = MRI.getType(I.getOperand(0).getReg()); |
| if (!Ty.isValid()) { |
| LLVM_DEBUG(dbgs() << "Generic binop register should be typed\n"); |
| return true; |
| } |
| |
| const RegisterBank *PrevOpBank = nullptr; |
| for (auto &MO : I.operands()) { |
| // FIXME: Support non-register operands. |
| if (!MO.isReg()) { |
| LLVM_DEBUG(dbgs() << "Generic inst non-reg operands are unsupported\n"); |
| return true; |
| } |
| |
| // FIXME: Can generic operations have physical registers operands? If |
| // so, this will need to be taught about that, and we'll need to get the |
| // bank out of the minimal class for the register. |
| // Either way, this needs to be documented (and possibly verified). |
| if (!Register::isVirtualRegister(MO.getReg())) { |
| LLVM_DEBUG(dbgs() << "Generic inst has physical register operand\n"); |
| return true; |
| } |
| |
| const RegisterBank *OpBank = RBI.getRegBank(MO.getReg(), MRI, TRI); |
| if (!OpBank) { |
| LLVM_DEBUG(dbgs() << "Generic register has no bank or class\n"); |
| return true; |
| } |
| |
| if (PrevOpBank && OpBank != PrevOpBank) { |
| LLVM_DEBUG(dbgs() << "Generic inst operands have different banks\n"); |
| return true; |
| } |
| PrevOpBank = OpBank; |
| } |
| return false; |
| } |
| |
| /// Select the AArch64 opcode for the basic binary operation \p GenericOpc |
| /// (such as G_OR or G_SDIV), appropriate for the register bank \p RegBankID |
| /// and of size \p OpSize. |
| /// \returns \p GenericOpc if the combination is unsupported. |
| static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID, |
| unsigned OpSize) { |
| switch (RegBankID) { |
| case AArch64::GPRRegBankID: |
| if (OpSize == 32) { |
| switch (GenericOpc) { |
| case TargetOpcode::G_SHL: |
| return AArch64::LSLVWr; |
| case TargetOpcode::G_LSHR: |
| return AArch64::LSRVWr; |
| case TargetOpcode::G_ASHR: |
| return AArch64::ASRVWr; |
| default: |
| return GenericOpc; |
| } |
| } else if (OpSize == 64) { |
| switch (GenericOpc) { |
| case TargetOpcode::G_PTR_ADD: |
| return AArch64::ADDXrr; |
| case TargetOpcode::G_SHL: |
| return AArch64::LSLVXr; |
| case TargetOpcode::G_LSHR: |
| return AArch64::LSRVXr; |
| case TargetOpcode::G_ASHR: |
| return AArch64::ASRVXr; |
| default: |
| return GenericOpc; |
| } |
| } |
| break; |
| case AArch64::FPRRegBankID: |
| switch (OpSize) { |
| case 32: |
| switch (GenericOpc) { |
| case TargetOpcode::G_FADD: |
| return AArch64::FADDSrr; |
| case TargetOpcode::G_FSUB: |
| return AArch64::FSUBSrr; |
| case TargetOpcode::G_FMUL: |
| return AArch64::FMULSrr; |
| case TargetOpcode::G_FDIV: |
| return AArch64::FDIVSrr; |
| default: |
| return GenericOpc; |
| } |
| case 64: |
| switch (GenericOpc) { |
| case TargetOpcode::G_FADD: |
| return AArch64::FADDDrr; |
| case TargetOpcode::G_FSUB: |
| return AArch64::FSUBDrr; |
| case TargetOpcode::G_FMUL: |
| return AArch64::FMULDrr; |
| case TargetOpcode::G_FDIV: |
| return AArch64::FDIVDrr; |
| case TargetOpcode::G_OR: |
| return AArch64::ORRv8i8; |
| default: |
| return GenericOpc; |
| } |
| } |
| break; |
| } |
| return GenericOpc; |
| } |
| |
| /// Select the AArch64 opcode for the G_LOAD or G_STORE operation \p GenericOpc, |
| /// appropriate for the (value) register bank \p RegBankID and of memory access |
| /// size \p OpSize. This returns the variant with the base+unsigned-immediate |
| /// addressing mode (e.g., LDRXui). |
| /// \returns \p GenericOpc if the combination is unsupported. |
| static unsigned selectLoadStoreUIOp(unsigned GenericOpc, unsigned RegBankID, |
| unsigned OpSize) { |
| const bool isStore = GenericOpc == TargetOpcode::G_STORE; |
| switch (RegBankID) { |
| case AArch64::GPRRegBankID: |
| switch (OpSize) { |
| case 8: |
| return isStore ? AArch64::STRBBui : AArch64::LDRBBui; |
| case 16: |
| return isStore ? AArch64::STRHHui : AArch64::LDRHHui; |
| case 32: |
| return isStore ? AArch64::STRWui : AArch64::LDRWui; |
| case 64: |
| return isStore ? AArch64::STRXui : AArch64::LDRXui; |
| } |
| break; |
| case AArch64::FPRRegBankID: |
| switch (OpSize) { |
| case 8: |
| return isStore ? AArch64::STRBui : AArch64::LDRBui; |
| case 16: |
| return isStore ? AArch64::STRHui : AArch64::LDRHui; |
| case 32: |
| return isStore ? AArch64::STRSui : AArch64::LDRSui; |
| case 64: |
| return isStore ? AArch64::STRDui : AArch64::LDRDui; |
| } |
| break; |
| } |
| return GenericOpc; |
| } |
| |
| #ifndef NDEBUG |
| /// Helper function that verifies that we have a valid copy at the end of |
| /// selectCopy. Verifies that the source and dest have the expected sizes and |
| /// then returns true. |
| static bool isValidCopy(const MachineInstr &I, const RegisterBank &DstBank, |
| const MachineRegisterInfo &MRI, |
| const TargetRegisterInfo &TRI, |
| const RegisterBankInfo &RBI) { |
| const Register DstReg = I.getOperand(0).getReg(); |
| const Register SrcReg = I.getOperand(1).getReg(); |
| const unsigned DstSize = RBI.getSizeInBits(DstReg, MRI, TRI); |
| const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI); |
| |
| // Make sure the size of the source and dest line up. |
| assert( |
| (DstSize == SrcSize || |
| // Copies are a mean to setup initial types, the number of |
| // bits may not exactly match. |
| (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || |
| // Copies are a mean to copy bits around, as long as we are |
| // on the same register class, that's fine. Otherwise, that |
| // means we need some SUBREG_TO_REG or AND & co. |
| (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && |
| "Copy with different width?!"); |
| |
| // Check the size of the destination. |
| assert((DstSize <= 64 || DstBank.getID() == AArch64::FPRRegBankID) && |
| "GPRs cannot get more than 64-bit width values"); |
| |
| return true; |
| } |
| #endif |
| |
| /// Helper function for selectCopy. Inserts a subregister copy from |
| /// \p *From to \p *To, linking it up to \p I. |
| /// |
| /// e.g, given I = "Dst = COPY SrcReg", we'll transform that into |
| /// |
| /// CopyReg (From class) = COPY SrcReg |
| /// SubRegCopy (To class) = COPY CopyReg:SubReg |
| /// Dst = COPY SubRegCopy |
| static bool selectSubregisterCopy(MachineInstr &I, MachineRegisterInfo &MRI, |
| const RegisterBankInfo &RBI, Register SrcReg, |
| const TargetRegisterClass *From, |
| const TargetRegisterClass *To, |
| unsigned SubReg) { |
| MachineIRBuilder MIB(I); |
| auto Copy = MIB.buildCopy({From}, {SrcReg}); |
| auto SubRegCopy = MIB.buildInstr(TargetOpcode::COPY, {To}, {}) |
| .addReg(Copy.getReg(0), 0, SubReg); |
| MachineOperand &RegOp = I.getOperand(1); |
| RegOp.setReg(SubRegCopy.getReg(0)); |
| |
| // It's possible that the destination register won't be constrained. Make |
| // sure that happens. |
| if (!Register::isPhysicalRegister(I.getOperand(0).getReg())) |
| RBI.constrainGenericRegister(I.getOperand(0).getReg(), *To, MRI); |
| |
| return true; |
| } |
| |
| /// Helper function to get the source and destination register classes for a |
| /// copy. Returns a std::pair containing the source register class for the |
| /// copy, and the destination register class for the copy. If a register class |
| /// cannot be determined, then it will be nullptr. |
| static std::pair<const TargetRegisterClass *, const TargetRegisterClass *> |
| getRegClassesForCopy(MachineInstr &I, const TargetInstrInfo &TII, |
| MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, |
| const RegisterBankInfo &RBI) { |
| Register DstReg = I.getOperand(0).getReg(); |
| Register SrcReg = I.getOperand(1).getReg(); |
| const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI); |
| const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI); |
| unsigned DstSize = RBI.getSizeInBits(DstReg, MRI, TRI); |
| unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI); |
| |
| // Special casing for cross-bank copies of s1s. We can technically represent |
| // a 1-bit value with any size of register. The minimum size for a GPR is 32 |
| // bits. So, we need to put the FPR on 32 bits as well. |
| // |
| // FIXME: I'm not sure if this case holds true outside of copies. If it does, |
| // then we can pull it into the helpers that get the appropriate class for a |
| // register bank. Or make a new helper that carries along some constraint |
| // information. |
| if (SrcRegBank != DstRegBank && (DstSize == 1 && SrcSize == 1)) |
| SrcSize = DstSize = 32; |
| |
| return {getMinClassForRegBank(SrcRegBank, SrcSize, true), |
| getMinClassForRegBank(DstRegBank, DstSize, true)}; |
| } |
| |
| static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII, |
| MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, |
| const RegisterBankInfo &RBI) { |
| |
| Register DstReg = I.getOperand(0).getReg(); |
| Register SrcReg = I.getOperand(1).getReg(); |
| const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI); |
| const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI); |
| |
| // Find the correct register classes for the source and destination registers. |
| const TargetRegisterClass *SrcRC; |
| const TargetRegisterClass *DstRC; |
| std::tie(SrcRC, DstRC) = getRegClassesForCopy(I, TII, MRI, TRI, RBI); |
| |
| if (!DstRC) { |
| LLVM_DEBUG(dbgs() << "Unexpected dest size " |
| << RBI.getSizeInBits(DstReg, MRI, TRI) << '\n'); |
| return false; |
| } |
| |
| // A couple helpers below, for making sure that the copy we produce is valid. |
| |
| // Set to true if we insert a SUBREG_TO_REG. If we do this, then we don't want |
| // to verify that the src and dst are the same size, since that's handled by |
| // the SUBREG_TO_REG. |
| bool KnownValid = false; |
| |
| // Returns true, or asserts if something we don't expect happens. Instead of |
| // returning true, we return isValidCopy() to ensure that we verify the |
| // result. |
| auto CheckCopy = [&]() { |
| // If we have a bitcast or something, we can't have physical registers. |
| assert((I.isCopy() || |
| (!Register::isPhysicalRegister(I.getOperand(0).getReg()) && |
| !Register::isPhysicalRegister(I.getOperand(1).getReg()))) && |
| "No phys reg on generic operator!"); |
| assert(KnownValid || isValidCopy(I, DstRegBank, MRI, TRI, RBI)); |
| (void)KnownValid; |
| return true; |
| }; |
| |
| // Is this a copy? If so, then we may need to insert a subregister copy, or |
| // a SUBREG_TO_REG. |
| if (I.isCopy()) { |
| // Yes. Check if there's anything to fix up. |
| if (!SrcRC) { |
| LLVM_DEBUG(dbgs() << "Couldn't determine source register class\n"); |
| return false; |
| } |
| |
| unsigned SrcSize = TRI.getRegSizeInBits(*SrcRC); |
| unsigned DstSize = TRI.getRegSizeInBits(*DstRC); |
| |
| // If we're doing a cross-bank copy on different-sized registers, we need |
| // to do a bit more work. |
| if (SrcSize > DstSize) { |
| // We're doing a cross-bank copy into a smaller register. We need a |
| // subregister copy. First, get a register class that's on the same bank |
| // as the destination, but the same size as the source. |
| const TargetRegisterClass *SubregRC = |
| getMinClassForRegBank(DstRegBank, SrcSize, true); |
| assert(SubregRC && "Didn't get a register class for subreg?"); |
| |
| // Get the appropriate subregister for the destination. |
| unsigned SubReg = 0; |
| if (!getSubRegForClass(DstRC, TRI, SubReg)) { |
| LLVM_DEBUG(dbgs() << "Couldn't determine subregister for copy.\n"); |
| return false; |
| } |
| |
| // Now, insert a subregister copy using the new register class. |
| selectSubregisterCopy(I, MRI, RBI, SrcReg, SubregRC, DstRC, SubReg); |
| return CheckCopy(); |
| } |
| |
| // Is this a cross-bank copy? |
| if (DstRegBank.getID() != SrcRegBank.getID()) { |
| if (DstRegBank.getID() == AArch64::GPRRegBankID && DstSize == 32 && |
| SrcSize == 16) { |
| // Special case for FPR16 to GPR32. |
| // FIXME: This can probably be generalized like the above case. |
| Register PromoteReg = |
| MRI.createVirtualRegister(&AArch64::FPR32RegClass); |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), |
| TII.get(AArch64::SUBREG_TO_REG), PromoteReg) |
| .addImm(0) |
| .addUse(SrcReg) |
| .addImm(AArch64::hsub); |
| MachineOperand &RegOp = I.getOperand(1); |
| RegOp.setReg(PromoteReg); |
| |
| // Promise that the copy is implicitly validated by the SUBREG_TO_REG. |
| KnownValid = true; |
| } |
| } |
| |
| // If the destination is a physical register, then there's nothing to |
| // change, so we're done. |
| if (Register::isPhysicalRegister(DstReg)) |
| return CheckCopy(); |
| } |
| |
| // No need to constrain SrcReg. It will get constrained when we hit another |
| // of its use or its defs. Copies do not have constraints. |
| if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { |
| LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode()) |
| << " operand\n"); |
| return false; |
| } |
| I.setDesc(TII.get(AArch64::COPY)); |
| return CheckCopy(); |
| } |
| |
| static unsigned selectFPConvOpc(unsigned GenericOpc, LLT DstTy, LLT SrcTy) { |
| if (!DstTy.isScalar() || !SrcTy.isScalar()) |
| return GenericOpc; |
| |
| const unsigned DstSize = DstTy.getSizeInBits(); |
| const unsigned SrcSize = SrcTy.getSizeInBits(); |
| |
| switch (DstSize) { |
| case 32: |
| switch (SrcSize) { |
| case 32: |
| switch (GenericOpc) { |
| case TargetOpcode::G_SITOFP: |
| return AArch64::SCVTFUWSri; |
| case TargetOpcode::G_UITOFP: |
| return AArch64::UCVTFUWSri; |
| case TargetOpcode::G_FPTOSI: |
| return AArch64::FCVTZSUWSr; |
| case TargetOpcode::G_FPTOUI: |
| return AArch64::FCVTZUUWSr; |
| default: |
| return GenericOpc; |
| } |
| case 64: |
| switch (GenericOpc) { |
| case TargetOpcode::G_SITOFP: |
| return AArch64::SCVTFUXSri; |
| case TargetOpcode::G_UITOFP: |
| return AArch64::UCVTFUXSri; |
| case TargetOpcode::G_FPTOSI: |
| return AArch64::FCVTZSUWDr; |
| case TargetOpcode::G_FPTOUI: |
| return AArch64::FCVTZUUWDr; |
| default: |
| return GenericOpc; |
| } |
| default: |
| return GenericOpc; |
| } |
| case 64: |
| switch (SrcSize) { |
| case 32: |
| switch (GenericOpc) { |
| case TargetOpcode::G_SITOFP: |
| return AArch64::SCVTFUWDri; |
| case TargetOpcode::G_UITOFP: |
| return AArch64::UCVTFUWDri; |
| case TargetOpcode::G_FPTOSI: |
| return AArch64::FCVTZSUXSr; |
| case TargetOpcode::G_FPTOUI: |
| return AArch64::FCVTZUUXSr; |
| default: |
| return GenericOpc; |
| } |
| case 64: |
| switch (GenericOpc) { |
| case TargetOpcode::G_SITOFP: |
| return AArch64::SCVTFUXDri; |
| case TargetOpcode::G_UITOFP: |
| return AArch64::UCVTFUXDri; |
| case TargetOpcode::G_FPTOSI: |
| return AArch64::FCVTZSUXDr; |
| case TargetOpcode::G_FPTOUI: |
| return AArch64::FCVTZUUXDr; |
| default: |
| return GenericOpc; |
| } |
| default: |
| return GenericOpc; |
| } |
| default: |
| return GenericOpc; |
| }; |
| return GenericOpc; |
| } |
| |
| static unsigned selectSelectOpc(MachineInstr &I, MachineRegisterInfo &MRI, |
| const RegisterBankInfo &RBI) { |
| const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo(); |
| bool IsFP = (RBI.getRegBank(I.getOperand(0).getReg(), MRI, TRI)->getID() != |
| AArch64::GPRRegBankID); |
| LLT Ty = MRI.getType(I.getOperand(0).getReg()); |
| if (Ty == LLT::scalar(32)) |
| return IsFP ? AArch64::FCSELSrrr : AArch64::CSELWr; |
| else if (Ty == LLT::scalar(64) || Ty == LLT::pointer(0, 64)) |
| return IsFP ? AArch64::FCSELDrrr : AArch64::CSELXr; |
| return 0; |
| } |
| |
| /// Helper function to select the opcode for a G_FCMP. |
| static unsigned selectFCMPOpc(MachineInstr &I, MachineRegisterInfo &MRI) { |
| // If this is a compare against +0.0, then we don't have to explicitly |
| // materialize a constant. |
| const ConstantFP *FPImm = getConstantFPVRegVal(I.getOperand(3).getReg(), MRI); |
| bool ShouldUseImm = FPImm && (FPImm->isZero() && !FPImm->isNegative()); |
| unsigned OpSize = MRI.getType(I.getOperand(2).getReg()).getSizeInBits(); |
| if (OpSize != 32 && OpSize != 64) |
| return 0; |
| unsigned CmpOpcTbl[2][2] = {{AArch64::FCMPSrr, AArch64::FCMPDrr}, |
| {AArch64::FCMPSri, AArch64::FCMPDri}}; |
| return CmpOpcTbl[ShouldUseImm][OpSize == 64]; |
| } |
| |
| /// Returns true if \p P is an unsigned integer comparison predicate. |
| static bool isUnsignedICMPPred(const CmpInst::Predicate P) { |
| switch (P) { |
| default: |
| return false; |
| case CmpInst::ICMP_UGT: |
| case CmpInst::ICMP_UGE: |
| case CmpInst::ICMP_ULT: |
| case CmpInst::ICMP_ULE: |
| return true; |
| } |
| } |
| |
| static AArch64CC::CondCode changeICMPPredToAArch64CC(CmpInst::Predicate P) { |
| switch (P) { |
| default: |
| llvm_unreachable("Unknown condition code!"); |
| case CmpInst::ICMP_NE: |
| return AArch64CC::NE; |
| case CmpInst::ICMP_EQ: |
| return AArch64CC::EQ; |
| case CmpInst::ICMP_SGT: |
| return AArch64CC::GT; |
| case CmpInst::ICMP_SGE: |
| return AArch64CC::GE; |
| case CmpInst::ICMP_SLT: |
| return AArch64CC::LT; |
| case CmpInst::ICMP_SLE: |
| return AArch64CC::LE; |
| case CmpInst::ICMP_UGT: |
| return AArch64CC::HI; |
| case CmpInst::ICMP_UGE: |
| return AArch64CC::HS; |
| case CmpInst::ICMP_ULT: |
| return AArch64CC::LO; |
| case CmpInst::ICMP_ULE: |
| return AArch64CC::LS; |
| } |
| } |
| |
| static void changeFCMPPredToAArch64CC(CmpInst::Predicate P, |
| AArch64CC::CondCode &CondCode, |
| AArch64CC::CondCode &CondCode2) { |
| CondCode2 = AArch64CC::AL; |
| switch (P) { |
| default: |
| llvm_unreachable("Unknown FP condition!"); |
| case CmpInst::FCMP_OEQ: |
| CondCode = AArch64CC::EQ; |
| break; |
| case CmpInst::FCMP_OGT: |
| CondCode = AArch64CC::GT; |
| break; |
| case CmpInst::FCMP_OGE: |
| CondCode = AArch64CC::GE; |
| break; |
| case CmpInst::FCMP_OLT: |
| CondCode = AArch64CC::MI; |
| break; |
| case CmpInst::FCMP_OLE: |
| CondCode = AArch64CC::LS; |
| break; |
| case CmpInst::FCMP_ONE: |
| CondCode = AArch64CC::MI; |
| CondCode2 = AArch64CC::GT; |
| break; |
| case CmpInst::FCMP_ORD: |
| CondCode = AArch64CC::VC; |
| break; |
| case CmpInst::FCMP_UNO: |
| CondCode = AArch64CC::VS; |
| break; |
| case CmpInst::FCMP_UEQ: |
| CondCode = AArch64CC::EQ; |
| CondCode2 = AArch64CC::VS; |
| break; |
| case CmpInst::FCMP_UGT: |
| CondCode = AArch64CC::HI; |
| break; |
| case CmpInst::FCMP_UGE: |
| CondCode = AArch64CC::PL; |
| break; |
| case CmpInst::FCMP_ULT: |
| CondCode = AArch64CC::LT; |
| break; |
| case CmpInst::FCMP_ULE: |
| CondCode = AArch64CC::LE; |
| break; |
| case CmpInst::FCMP_UNE: |
| CondCode = AArch64CC::NE; |
| break; |
| } |
| } |
| |
| bool AArch64InstructionSelector::selectCompareBranch( |
| MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const { |
| |
| const Register CondReg = I.getOperand(0).getReg(); |
| MachineBasicBlock *DestMBB = I.getOperand(1).getMBB(); |
| MachineInstr *CCMI = MRI.getVRegDef(CondReg); |
| if (CCMI->getOpcode() == TargetOpcode::G_TRUNC) |
| CCMI = MRI.getVRegDef(CCMI->getOperand(1).getReg()); |
| if (CCMI->getOpcode() != TargetOpcode::G_ICMP) |
| return false; |
| |
| Register LHS = CCMI->getOperand(2).getReg(); |
| Register RHS = CCMI->getOperand(3).getReg(); |
| auto VRegAndVal = getConstantVRegValWithLookThrough(RHS, MRI); |
| if (!VRegAndVal) |
| std::swap(RHS, LHS); |
| |
| VRegAndVal = getConstantVRegValWithLookThrough(RHS, MRI); |
| if (!VRegAndVal || VRegAndVal->Value != 0) { |
| MachineIRBuilder MIB(I); |
| // If we can't select a CBZ then emit a cmp + Bcc. |
| if (!emitIntegerCompare(CCMI->getOperand(2), CCMI->getOperand(3), |
| CCMI->getOperand(1), MIB)) |
| return false; |
| const AArch64CC::CondCode CC = changeICMPPredToAArch64CC( |
| (CmpInst::Predicate)CCMI->getOperand(1).getPredicate()); |
| MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC).addMBB(DestMBB); |
| I.eraseFromParent(); |
| return true; |
| } |
| |
| const RegisterBank &RB = *RBI.getRegBank(LHS, MRI, TRI); |
| if (RB.getID() != AArch64::GPRRegBankID) |
| return false; |
| |
| const auto Pred = (CmpInst::Predicate)CCMI->getOperand(1).getPredicate(); |
| if (Pred != CmpInst::ICMP_NE && Pred != CmpInst::ICMP_EQ) |
| return false; |
| |
| const unsigned CmpWidth = MRI.getType(LHS).getSizeInBits(); |
| unsigned CBOpc = 0; |
| if (CmpWidth <= 32) |
| CBOpc = (Pred == CmpInst::ICMP_EQ ? AArch64::CBZW : AArch64::CBNZW); |
| else if (CmpWidth == 64) |
| CBOpc = (Pred == CmpInst::ICMP_EQ ? AArch64::CBZX : AArch64::CBNZX); |
| else |
| return false; |
| |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CBOpc)) |
| .addUse(LHS) |
| .addMBB(DestMBB) |
| .constrainAllUses(TII, TRI, RBI); |
| |
| I.eraseFromParent(); |
| return true; |
| } |
| |
| /// Returns the element immediate value of a vector shift operand if found. |
| /// This needs to detect a splat-like operation, e.g. a G_BUILD_VECTOR. |
| static Optional<int64_t> getVectorShiftImm(Register Reg, |
| MachineRegisterInfo &MRI) { |
| assert(MRI.getType(Reg).isVector() && "Expected a *vector* shift operand"); |
| MachineInstr *OpMI = MRI.getVRegDef(Reg); |
| assert(OpMI && "Expected to find a vreg def for vector shift operand"); |
| if (OpMI->getOpcode() != TargetOpcode::G_BUILD_VECTOR) |
| return None; |
| |
| // Check all operands are identical immediates. |
| int64_t ImmVal = 0; |
| for (unsigned Idx = 1; Idx < OpMI->getNumOperands(); ++Idx) { |
| auto VRegAndVal = getConstantVRegValWithLookThrough(OpMI->getOperand(Idx).getReg(), MRI); |
| if (!VRegAndVal) |
| return None; |
| |
| if (Idx == 1) |
| ImmVal = VRegAndVal->Value; |
| if (ImmVal != VRegAndVal->Value) |
| return None; |
| } |
| |
| return ImmVal; |
| } |
| |
| /// Matches and returns the shift immediate value for a SHL instruction given |
| /// a shift operand. |
| static Optional<int64_t> getVectorSHLImm(LLT SrcTy, Register Reg, MachineRegisterInfo &MRI) { |
| Optional<int64_t> ShiftImm = getVectorShiftImm(Reg, MRI); |
| if (!ShiftImm) |
| return None; |
| // Check the immediate is in range for a SHL. |
| int64_t Imm = *ShiftImm; |
| if (Imm < 0) |
| return None; |
| switch (SrcTy.getElementType().getSizeInBits()) { |
| default: |
| LLVM_DEBUG(dbgs() << "Unhandled element type for vector shift"); |
| return None; |
| case 8: |
| if (Imm > 7) |
| return None; |
| break; |
| case 16: |
| if (Imm > 15) |
| return None; |
| break; |
| case 32: |
| if (Imm > 31) |
| return None; |
| break; |
| case 64: |
| if (Imm > 63) |
| return None; |
| break; |
| } |
| return Imm; |
| } |
| |
| bool AArch64InstructionSelector::selectVectorSHL( |
| MachineInstr &I, MachineRegisterInfo &MRI) const { |
| assert(I.getOpcode() == TargetOpcode::G_SHL); |
| Register DstReg = I.getOperand(0).getReg(); |
| const LLT Ty = MRI.getType(DstReg); |
| Register Src1Reg = I.getOperand(1).getReg(); |
| Register Src2Reg = I.getOperand(2).getReg(); |
| |
| if (!Ty.isVector()) |
| return false; |
| |
| // Check if we have a vector of constants on RHS that we can select as the |
| // immediate form. |
| Optional<int64_t> ImmVal = getVectorSHLImm(Ty, Src2Reg, MRI); |
| |
| unsigned Opc = 0; |
| if (Ty == LLT::vector(2, 64)) { |
| Opc = ImmVal ? AArch64::SHLv2i64_shift : AArch64::USHLv2i64; |
| } else if (Ty == LLT::vector(4, 32)) { |
| Opc = ImmVal ? AArch64::SHLv4i32_shift : AArch64::USHLv4i32; |
| } else if (Ty == LLT::vector(2, 32)) { |
| Opc = ImmVal ? AArch64::SHLv2i32_shift : AArch64::USHLv2i32; |
| } else { |
| LLVM_DEBUG(dbgs() << "Unhandled G_SHL type"); |
| return false; |
| } |
| |
| MachineIRBuilder MIB(I); |
| auto Shl = MIB.buildInstr(Opc, {DstReg}, {Src1Reg}); |
| if (ImmVal) |
| Shl.addImm(*ImmVal); |
| else |
| Shl.addUse(Src2Reg); |
| constrainSelectedInstRegOperands(*Shl, TII, TRI, RBI); |
| I.eraseFromParent(); |
| return true; |
| } |
| |
| bool AArch64InstructionSelector::selectVectorASHR( |
| MachineInstr &I, MachineRegisterInfo &MRI) const { |
| assert(I.getOpcode() == TargetOpcode::G_ASHR); |
| Register DstReg = I.getOperand(0).getReg(); |
| const LLT Ty = MRI.getType(DstReg); |
| Register Src1Reg = I.getOperand(1).getReg(); |
| Register Src2Reg = I.getOperand(2).getReg(); |
| |
| if (!Ty.isVector()) |
| return false; |
| |
| // There is not a shift right register instruction, but the shift left |
| // register instruction takes a signed value, where negative numbers specify a |
| // right shift. |
| |
| unsigned Opc = 0; |
| unsigned NegOpc = 0; |
| const TargetRegisterClass *RC = nullptr; |
| if (Ty == LLT::vector(2, 64)) { |
| Opc = AArch64::SSHLv2i64; |
| NegOpc = AArch64::NEGv2i64; |
| RC = &AArch64::FPR128RegClass; |
| } else if (Ty == LLT::vector(4, 32)) { |
| Opc = AArch64::SSHLv4i32; |
| NegOpc = AArch64::NEGv4i32; |
| RC = &AArch64::FPR128RegClass; |
| } else if (Ty == LLT::vector(2, 32)) { |
| Opc = AArch64::SSHLv2i32; |
| NegOpc = AArch64::NEGv2i32; |
| RC = &AArch64::FPR64RegClass; |
| } else { |
| LLVM_DEBUG(dbgs() << "Unhandled G_ASHR type"); |
| return false; |
| } |
| |
| MachineIRBuilder MIB(I); |
| auto Neg = MIB.buildInstr(NegOpc, {RC}, {Src2Reg}); |
| constrainSelectedInstRegOperands(*Neg, TII, TRI, RBI); |
| auto SShl = MIB.buildInstr(Opc, {DstReg}, {Src1Reg, Neg}); |
| constrainSelectedInstRegOperands(*SShl, TII, TRI, RBI); |
| I.eraseFromParent(); |
| return true; |
| } |
| |
| bool AArch64InstructionSelector::selectVaStartAAPCS( |
| MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const { |
| return false; |
| } |
| |
| bool AArch64InstructionSelector::selectVaStartDarwin( |
| MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const { |
| AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>(); |
| Register ListReg = I.getOperand(0).getReg(); |
| |
| Register ArgsAddrReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass); |
| |
| auto MIB = |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::ADDXri)) |
| .addDef(ArgsAddrReg) |
| .addFrameIndex(FuncInfo->getVarArgsStackIndex()) |
| .addImm(0) |
| .addImm(0); |
| |
| constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI); |
| |
| MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::STRXui)) |
| .addUse(ArgsAddrReg) |
| .addUse(ListReg) |
| .addImm(0) |
| .addMemOperand(*I.memoperands_begin()); |
| |
| constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI); |
| I.eraseFromParent(); |
| return true; |
| } |
| |
| void AArch64InstructionSelector::materializeLargeCMVal( |
| MachineInstr &I, const Value *V, unsigned OpFlags) const { |
| MachineBasicBlock &MBB = *I.getParent(); |
| MachineFunction &MF = *MBB.getParent(); |
| MachineRegisterInfo &MRI = MF.getRegInfo(); |
| MachineIRBuilder MIB(I); |
| |
| auto MovZ = MIB.buildInstr(AArch64::MOVZXi, {&AArch64::GPR64RegClass}, {}); |
| MovZ->addOperand(MF, I.getOperand(1)); |
| MovZ->getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_G0 | |
| AArch64II::MO_NC); |
| MovZ->addOperand(MF, MachineOperand::CreateImm(0)); |
| constrainSelectedInstRegOperands(*MovZ, TII, TRI, RBI); |
| |
| auto BuildMovK = [&](Register SrcReg, unsigned char Flags, unsigned Offset, |
| Register ForceDstReg) { |
| Register DstReg = ForceDstReg |
| ? ForceDstReg |
| : MRI.createVirtualRegister(&AArch64::GPR64RegClass); |
| auto MovI = MIB.buildInstr(AArch64::MOVKXi).addDef(DstReg).addUse(SrcReg); |
| if (auto *GV = dyn_cast<GlobalValue>(V)) { |
| MovI->addOperand(MF, MachineOperand::CreateGA( |
| GV, MovZ->getOperand(1).getOffset(), Flags)); |
| } else { |
| MovI->addOperand( |
| MF, MachineOperand::CreateBA(cast<BlockAddress>(V), |
| MovZ->getOperand(1).getOffset(), Flags)); |
| } |
| MovI->addOperand(MF, MachineOperand::CreateImm(Offset)); |
| constrainSelectedInstRegOperands(*MovI, TII, TRI, RBI); |
| return DstReg; |
| }; |
| Register DstReg = BuildMovK(MovZ.getReg(0), |
| AArch64II::MO_G1 | AArch64II::MO_NC, 16, 0); |
| DstReg = BuildMovK(DstReg, AArch64II::MO_G2 | AArch64II::MO_NC, 32, 0); |
| BuildMovK(DstReg, AArch64II::MO_G3, 48, I.getOperand(0).getReg()); |
| return; |
| } |
| |
| void AArch64InstructionSelector::preISelLower(MachineInstr &I) const { |
| MachineBasicBlock &MBB = *I.getParent(); |
| MachineFunction &MF = *MBB.getParent(); |
| MachineRegisterInfo &MRI = MF.getRegInfo(); |
| |
| switch (I.getOpcode()) { |
| case TargetOpcode::G_SHL: |
| case TargetOpcode::G_ASHR: |
| case TargetOpcode::G_LSHR: { |
| // These shifts are legalized to have 64 bit shift amounts because we want |
| // to take advantage of the existing imported selection patterns that assume |
| // the immediates are s64s. However, if the shifted type is 32 bits and for |
| // some reason we receive input GMIR that has an s64 shift amount that's not |
| // a G_CONSTANT, insert a truncate so that we can still select the s32 |
| // register-register variant. |
| Register SrcReg = I.getOperand(1).getReg(); |
| Register ShiftReg = I.getOperand(2).getReg(); |
| const LLT ShiftTy = MRI.getType(ShiftReg); |
| const LLT SrcTy = MRI.getType(SrcReg); |
| if (SrcTy.isVector()) |
| return; |
| assert(!ShiftTy.isVector() && "unexpected vector shift ty"); |
| if (SrcTy.getSizeInBits() != 32 || ShiftTy.getSizeInBits() != 64) |
| return; |
| auto *AmtMI = MRI.getVRegDef(ShiftReg); |
| assert(AmtMI && "could not find a vreg definition for shift amount"); |
| if (AmtMI->getOpcode() != TargetOpcode::G_CONSTANT) { |
| // Insert a subregister copy to implement a 64->32 trunc |
| MachineIRBuilder MIB(I); |
| auto Trunc = MIB.buildInstr(TargetOpcode::COPY, {SrcTy}, {}) |
| .addReg(ShiftReg, 0, AArch64::sub_32); |
| MRI.setRegBank(Trunc.getReg(0), RBI.getRegBank(AArch64::GPRRegBankID)); |
| I.getOperand(2).setReg(Trunc.getReg(0)); |
| } |
| return; |
| } |
| case TargetOpcode::G_STORE: |
| contractCrossBankCopyIntoStore(I, MRI); |
| return; |
| default: |
| return; |
| } |
| } |
| |
| bool AArch64InstructionSelector::earlySelectSHL( |
| MachineInstr &I, MachineRegisterInfo &MRI) const { |
| // We try to match the immediate variant of LSL, which is actually an alias |
| // for a special case of UBFM. Otherwise, we fall back to the imported |
| // selector which will match the register variant. |
| assert(I.getOpcode() == TargetOpcode::G_SHL && "unexpected op"); |
| const auto &MO = I.getOperand(2); |
| auto VRegAndVal = getConstantVRegVal(MO.getReg(), MRI); |
| if (!VRegAndVal) |
| return false; |
| |
| const LLT DstTy = MRI.getType(I.getOperand(0).getReg()); |
| if (DstTy.isVector()) |
| return false; |
| bool Is64Bit = DstTy.getSizeInBits() == 64; |
| auto Imm1Fn = Is64Bit ? selectShiftA_64(MO) : selectShiftA_32(MO); |
| auto Imm2Fn = Is64Bit ? selectShiftB_64(MO) : selectShiftB_32(MO); |
| MachineIRBuilder MIB(I); |
| |
| if (!Imm1Fn || !Imm2Fn) |
| return false; |
| |
| auto NewI = |
| MIB.buildInstr(Is64Bit ? AArch64::UBFMXri : AArch64::UBFMWri, |
| {I.getOperand(0).getReg()}, {I.getOperand(1).getReg()}); |
| |
| for (auto &RenderFn : *Imm1Fn) |
| RenderFn(NewI); |
| for (auto &RenderFn : *Imm2Fn) |
| RenderFn(NewI); |
| |
| I.eraseFromParent(); |
| return constrainSelectedInstRegOperands(*NewI, TII, TRI, RBI); |
| } |
| |
| void AArch64InstructionSelector::contractCrossBankCopyIntoStore( |
| MachineInstr &I, MachineRegisterInfo &MRI) const { |
| assert(I.getOpcode() == TargetOpcode::G_STORE && "Expected G_STORE"); |
| // If we're storing a scalar, it doesn't matter what register bank that |
| // scalar is on. All that matters is the size. |
| // |
| // So, if we see something like this (with a 32-bit scalar as an example): |
| // |
| // %x:gpr(s32) = ... something ... |
| // %y:fpr(s32) = COPY %x:gpr(s32) |
| // G_STORE %y:fpr(s32) |
| // |
| // We can fix this up into something like this: |
| // |
| // G_STORE %x:gpr(s32) |
| // |
| // And then continue the selection process normally. |
| MachineInstr *Def = getDefIgnoringCopies(I.getOperand(0).getReg(), MRI); |
| if (!Def) |
| return; |
| Register DefDstReg = Def->getOperand(0).getReg(); |
| LLT DefDstTy = MRI.getType(DefDstReg); |
| Register StoreSrcReg = I.getOperand(0).getReg(); |
| LLT StoreSrcTy = MRI.getType(StoreSrcReg); |
| |
| // If we get something strange like a physical register, then we shouldn't |
| // go any further. |
| if (!DefDstTy.isValid()) |
| return; |
| |
| // Are the source and dst types the same size? |
| if (DefDstTy.getSizeInBits() != StoreSrcTy.getSizeInBits()) |
| return; |
| |
| if (RBI.getRegBank(StoreSrcReg, MRI, TRI) == |
| RBI.getRegBank(DefDstReg, MRI, TRI)) |
| return; |
| |
| // We have a cross-bank copy, which is entering a store. Let's fold it. |
| I.getOperand(0).setReg(DefDstReg); |
| } |
| |
| bool AArch64InstructionSelector::earlySelect(MachineInstr &I) const { |
| assert(I.getParent() && "Instruction should be in a basic block!"); |
| assert(I.getParent()->getParent() && "Instruction should be in a function!"); |
| |
| MachineBasicBlock &MBB = *I.getParent(); |
| MachineFunction &MF = *MBB.getParent(); |
| MachineRegisterInfo &MRI = MF.getRegInfo(); |
| |
| switch (I.getOpcode()) { |
| case TargetOpcode::G_SHL: |
| return earlySelectSHL(I, MRI); |
| case TargetOpcode::G_CONSTANT: { |
| bool IsZero = false; |
| if (I.getOperand(1).isCImm()) |
| IsZero = I.getOperand(1).getCImm()->getZExtValue() == 0; |
| else if (I.getOperand(1).isImm()) |
| IsZero = I.getOperand(1).getImm() == 0; |
| |
| if (!IsZero) |
| return false; |
| |
| Register DefReg = I.getOperand(0).getReg(); |
| LLT Ty = MRI.getType(DefReg); |
| if (Ty != LLT::scalar(64) && Ty != LLT::scalar(32)) |
| return false; |
| |
| if (Ty == LLT::scalar(64)) { |
| I.getOperand(1).ChangeToRegister(AArch64::XZR, false); |
| RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass, MRI); |
| } else { |
| I.getOperand(1).ChangeToRegister(AArch64::WZR, false); |
| RBI.constrainGenericRegister(DefReg, AArch64::GPR32RegClass, MRI); |
| } |
| I.setDesc(TII.get(TargetOpcode::COPY)); |
| return true; |
| } |
| default: |
| return false; |
| } |
| } |
| |
| bool AArch64InstructionSelector::select(MachineInstr &I) { |
| assert(I.getParent() && "Instruction should be in a basic block!"); |
| assert(I.getParent()->getParent() && "Instruction should be in a function!"); |
| |
| MachineBasicBlock &MBB = *I.getParent(); |
| MachineFunction &MF = *MBB.getParent(); |
| MachineRegisterInfo &MRI = MF.getRegInfo(); |
| |
| unsigned Opcode = I.getOpcode(); |
| // G_PHI requires same handling as PHI |
| if (!isPreISelGenericOpcode(Opcode) || Opcode == TargetOpcode::G_PHI) { |
| // Certain non-generic instructions also need some special handling. |
| |
| if (Opcode == TargetOpcode::LOAD_STACK_GUARD) |
| return constrainSelectedInstRegOperands(I, TII, TRI, RBI); |
| |
| if (Opcode == TargetOpcode::PHI || Opcode == TargetOpcode::G_PHI) { |
| const Register DefReg = I.getOperand(0).getReg(); |
| const LLT DefTy = MRI.getType(DefReg); |
| |
| const RegClassOrRegBank &RegClassOrBank = |
| MRI.getRegClassOrRegBank(DefReg); |
| |
| const TargetRegisterClass *DefRC |
| = RegClassOrBank.dyn_cast<const TargetRegisterClass *>(); |
| if (!DefRC) { |
| if (!DefTy.isValid()) { |
| LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n"); |
| return false; |
| } |
| const RegisterBank &RB = *RegClassOrBank.get<const RegisterBank *>(); |
| DefRC = getRegClassForTypeOnBank(DefTy, RB, RBI); |
| if (!DefRC) { |
| LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n"); |
| return false; |
| } |
| } |
| |
| I.setDesc(TII.get(TargetOpcode::PHI)); |
| |
| return RBI.constrainGenericRegister(DefReg, *DefRC, MRI); |
| } |
| |
| if (I.isCopy()) |
| return selectCopy(I, TII, MRI, TRI, RBI); |
| |
| return true; |
| } |
| |
| |
| if (I.getNumOperands() != I.getNumExplicitOperands()) { |
| LLVM_DEBUG( |
| dbgs() << "Generic instruction has unexpected implicit operands\n"); |
| return false; |
| } |
| |
| // Try to do some lowering before we start instruction selecting. These |
| // lowerings are purely transformations on the input G_MIR and so selection |
| // must continue after any modification of the instruction. |
| preISelLower(I); |
| |
| // There may be patterns where the importer can't deal with them optimally, |
| // but does select it to a suboptimal sequence so our custom C++ selection |
| // code later never has a chance to work on it. Therefore, we have an early |
| // selection attempt here to give priority to certain selection routines |
| // over the imported ones. |
| if (earlySelect(I)) |
| return true; |
| |
| if (selectImpl(I, *CoverageInfo)) |
| return true; |
| |
| LLT Ty = |
| I.getOperand(0).isReg() ? MRI.getType(I.getOperand(0).getReg()) : LLT{}; |
| |
| MachineIRBuilder MIB(I); |
| |
| switch (Opcode) { |
| case TargetOpcode::G_BRCOND: { |
| if (Ty.getSizeInBits() > 32) { |
| // We shouldn't need this on AArch64, but it would be implemented as an |
| // EXTRACT_SUBREG followed by a TBNZW because TBNZX has no encoding if the |
| // bit being tested is < 32. |
| LLVM_DEBUG(dbgs() << "G_BRCOND has type: " << Ty |
| << ", expected at most 32-bits"); |
| return false; |
| } |
| |
| const Register CondReg = I.getOperand(0).getReg(); |
| MachineBasicBlock *DestMBB = I.getOperand(1).getMBB(); |
| |
| // Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z |
| // instructions will not be produced, as they are conditional branch |
| // instructions that do not set flags. |
| bool ProduceNonFlagSettingCondBr = |
| !MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening); |
| if (ProduceNonFlagSettingCondBr && selectCompareBranch(I, MF, MRI)) |
| return true; |
| |
| if (ProduceNonFlagSettingCondBr) { |
| auto MIB = BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::TBNZW)) |
| .addUse(CondReg) |
| .addImm(/*bit offset=*/0) |
| .addMBB(DestMBB); |
| |
| I.eraseFromParent(); |
| return constrainSelectedInstRegOperands(*MIB.getInstr(), TII, TRI, RBI); |
| } else { |
| auto CMP = BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::ANDSWri)) |
| .addDef(AArch64::WZR) |
| .addUse(CondReg) |
| .addImm(1); |
| constrainSelectedInstRegOperands(*CMP.getInstr(), TII, TRI, RBI); |
| auto Bcc = |
| BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::Bcc)) |
| .addImm(AArch64CC::EQ) |
| .addMBB(DestMBB); |
| |
| I.eraseFromParent(); |
| return constrainSelectedInstRegOperands(*Bcc.getInstr(), TII, TRI, RBI); |
| } |
| } |
| |
| case TargetOpcode::G_BRINDIRECT: { |
| I.setDesc(TII.get(AArch64::BR)); |
| return constrainSelectedInstRegOperands(I, TII, TRI, RBI); |
| } |
| |
| case TargetOpcode::G_BRJT: |
| return selectBrJT(I, MRI); |
| |
| case TargetOpcode::G_BSWAP: { |
| // Handle vector types for G_BSWAP directly. |
| Register DstReg = I.getOperand(0).getReg(); |
| LLT DstTy = MRI.getType(DstReg); |
| |
| // We should only get vector types here; everything else is handled by the |
| // importer right now. |
| if (!DstTy.isVector() || DstTy.getSizeInBits() > 128) { |
| LLVM_DEBUG(dbgs() << "Dst type for G_BSWAP currently unsupported.\n"); |
| return false; |
| } |
| |
| // Only handle 4 and 2 element vectors for now. |
| // TODO: 16-bit elements. |
| unsigned NumElts = DstTy.getNumElements(); |
| if (NumElts != 4 && NumElts != 2) { |
| LLVM_DEBUG(dbgs() << "Unsupported number of elements for G_BSWAP.\n"); |
| return false; |
| } |
| |
| // Choose the correct opcode for the supported types. Right now, that's |
| // v2s32, v4s32, and v2s64. |
| unsigned Opc = 0; |
| unsigned EltSize = DstTy.getElementType().getSizeInBits(); |
| if (EltSize == 32) |
| Opc = (DstTy.getNumElements() == 2) ? AArch64::REV32v8i8 |
| : AArch64::REV32v16i8; |
| else if (EltSize == 64) |
| Opc = AArch64::REV64v16i8; |
| |
| // We should always get something by the time we get here... |
| assert(Opc != 0 && "Didn't get an opcode for G_BSWAP?"); |
| |
| I.setDesc(TII.get(Opc)); |
| return constrainSelectedInstRegOperands(I, TII, TRI, RBI); |
| } |
| |
| case TargetOpcode::G_FCONSTANT: |
| case TargetOpcode::G_CONSTANT: { |
| const bool isFP = Opcode == TargetOpcode::G_FCONSTANT; |
| |
| const LLT s8 = LLT::scalar(8); |
| const LLT s16 = LLT::scalar(16); |
| const LLT s32 = LLT::scalar(32); |
| const LLT s64 = LLT::scalar(64); |
| const LLT p0 = LLT::pointer(0, 64); |
| |
| const Register DefReg = I.getOperand(0).getReg(); |
| const LLT DefTy = MRI.getType(DefReg); |
| const unsigned DefSize = DefTy.getSizeInBits(); |
| const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI); |
| |
| // FIXME: Redundant check, but even less readable when factored out. |
| if (isFP) { |
| if (Ty != s32 && Ty != s64) { |
| LLVM_DEBUG(dbgs() << "Unable to materialize FP " << Ty |
| << " constant, expected: " << s32 << " or " << s64 |
| << '\n'); |
| return false; |
| } |
| |
| if (RB.getID() != AArch64::FPRRegBankID) { |
| LLVM_DEBUG(dbgs() << "Unable to materialize FP " << Ty |
| << " constant on bank: " << RB |
| << ", expected: FPR\n"); |
| return false; |
| } |
| |
| // The case when we have 0.0 is covered by tablegen. Reject it here so we |
| // can be sure tablegen works correctly and isn't rescued by this code. |
| if (I.getOperand(1).getFPImm()->getValueAPF().isExactlyValue(0.0)) |
| return false; |
| } else { |
| // s32 and s64 are covered by tablegen. |
| if (Ty != p0 && Ty != s8 && Ty != s16) { |
| LLVM_DEBUG(dbgs() << "Unable to materialize integer " << Ty |
| << " constant, expected: " << s32 << ", " << s64 |
| << ", or " << p0 << '\n'); |
| return false; |
| } |
| |
| if (RB.getID() != AArch64::GPRRegBankID) { |
| LLVM_DEBUG(dbgs() << "Unable to materialize integer " << Ty |
| << " constant on bank: " << RB |
| << ", expected: GPR\n"); |
| return false; |
| } |
| } |
| |
| // We allow G_CONSTANT of types < 32b. |
| const unsigned MovOpc = |
| DefSize == 64 ? AArch64::MOVi64imm : AArch64::MOVi32imm; |
| |
| if (isFP) { |
| // Either emit a FMOV, or emit a copy to emit a normal mov. |
| const TargetRegisterClass &GPRRC = |
| DefSize == 32 ? AArch64::GPR32RegClass : AArch64::GPR64RegClass; |
| const TargetRegisterClass &FPRRC = |
| DefSize == 32 ? AArch64::FPR32RegClass : AArch64::FPR64RegClass; |
| |
| // Can we use a FMOV instruction to represent the immediate? |
| if (emitFMovForFConstant(I, MRI)) |
| return true; |
| |
| // Nope. Emit a copy and use a normal mov instead. |
| const Register DefGPRReg = MRI.createVirtualRegister(&GPRRC); |
| MachineOperand &RegOp = I.getOperand(0); |
| RegOp.setReg(DefGPRReg); |
| MIB.setInsertPt(MIB.getMBB(), std::next(I.getIterator())); |
| MIB.buildCopy({DefReg}, {DefGPRReg}); |
| |
| if (!RBI.constrainGenericRegister(DefReg, FPRRC, MRI)) { |
| LLVM_DEBUG(dbgs() << "Failed to constrain G_FCONSTANT def operand\n"); |
| return false; |
| } |
| |
| MachineOperand &ImmOp = I.getOperand(1); |
| // FIXME: Is going through int64_t always correct? |
| ImmOp.ChangeToImmediate( |
| ImmOp.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue()); |
| } else if (I.getOperand(1).isCImm()) { |
| uint64_t Val = I.getOperand(1).getCImm()->getZExtValue(); |
| I.getOperand(1).ChangeToImmediate(Val); |
| } else if (I.getOperand(1).isImm()) { |
| uint64_t Val = I.getOperand(1).getImm(); |
| I.getOperand(1).ChangeToImmediate(Val); |
| } |
| |
| I.setDesc(TII.get(MovOpc)); |
| constrainSelectedInstRegOperands(I, TII, TRI, RBI); |
| return true; |
| } |
| case TargetOpcode::G_EXTRACT: { |
| Register DstReg = I.getOperand(0).getReg(); |
| Register SrcReg = I.getOperand(1).getReg(); |
| LLT SrcTy = MRI.getType(SrcReg); |
| LLT DstTy = MRI.getType(DstReg); |
| (void)DstTy; |
| unsigned SrcSize = SrcTy.getSizeInBits(); |
| |
| if (SrcTy.getSizeInBits() > 64) { |
| // This should be an extract of an s128, which is like a vector extract. |
| if (SrcTy.getSizeInBits() != 128) |
| return false; |
| // Only support extracting 64 bits from an s128 at the moment. |
| if (DstTy.getSizeInBits() != 64) |
| return false; |
| |
| const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI); |
| const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI); |
| // Check we have the right regbank always. |
| assert(SrcRB.getID() == AArch64::FPRRegBankID && |
| DstRB.getID() == AArch64::FPRRegBankID && |
| "Wrong extract regbank!"); |
| (void)SrcRB; |
| |
| // Emit the same code as a vector extract. |
| // Offset must be a multiple of 64. |
| unsigned Offset = I.getOperand(2).getImm(); |
| if (Offset % 64 != 0) |
| return false; |
| unsigned LaneIdx = Offset / 64; |
| MachineIRBuilder MIB(I); |
| MachineInstr *Extract = emitExtractVectorElt( |
| DstReg, DstRB, LLT::scalar(64), SrcReg, LaneIdx, MIB); |
| if (!Extract) |
| return false; |
| I.eraseFromParent(); |
| return true; |
| } |
| |
| I.setDesc(TII.get(SrcSize == 64 ? AArch64::UBFMXri : AArch64::UBFMWri)); |
| MachineInstrBuilder(MF, I).addImm(I.getOperand(2).getImm() + |
| Ty.getSizeInBits() - 1); |
| |
| if (SrcSize < 64) { |
| assert(SrcSize == 32 && DstTy.getSizeInBits() == 16 && |
| "unexpected G_EXTRACT types"); |
| return constrainSelectedInstRegOperands(I, TII, TRI, RBI); |
| } |
| |
| DstReg = MRI.createGenericVirtualRegister(LLT::scalar(64)); |
| MIB.setInsertPt(MIB.getMBB(), std::next(I.getIterator())); |
| MIB.buildInstr(TargetOpcode::COPY, {I.getOperand(0).getReg()}, {}) |
| .addReg(DstReg, 0, AArch64::sub_32); |
| RBI.constrainGenericRegister(I.getOperand(0).getReg(), |
| AArch64::GPR32RegClass, MRI); |
| I.getOperand(0).setReg(DstReg); |
| |
| return constrainSelectedInstRegOperands(I, TII, TRI, RBI); |
| } |
| |
| case TargetOpcode::G_INSERT: { |
| LLT SrcTy = MRI.getType(I.getOperand(2).getReg()); |
| LLT DstTy = MRI.getType(I.getOperand(0).getReg()); |
| unsigned DstSize = DstTy.getSizeInBits(); |
| // Larger inserts are vectors, same-size ones should be something else by |
| // now (split up or turned into COPYs). |
| if (Ty.getSizeInBits() > 64 || SrcTy.getSizeInBits() > 32) |
| return false; |
| |
| I.setDesc(TII.get(DstSize == 64 ? AArch64::BFMXri : AArch64::BFMWri)); |
| unsigned LSB = I.getOperand(3).getImm(); |
| unsigned Width = MRI.getType(I.getOperand(2).getReg()).getSizeInBits(); |
| I.getOperand(3).setImm((DstSize - LSB) % DstSize); |
| MachineInstrBuilder(MF, I).addImm(Width - 1); |
| |
| if (DstSize < 64) { |
| assert(DstSize == 32 && SrcTy.getSizeInBits() == 16 && |
| "unexpected G_INSERT types"); |
| return constrainSelectedInstRegOperands(I, TII, TRI, RBI); |
| } |
| |
| Register SrcReg = MRI.createGenericVirtualRegister(LLT::scalar(64)); |
| BuildMI(MBB, I.getIterator(), I.getDebugLoc(), |
| TII.get(AArch64::SUBREG_TO_REG)) |
| .addDef(SrcReg) |
| .addImm(0) |
| .addUse(I.getOperand(2).getReg()) |
| .addImm(AArch64::sub_32); |
| RBI.constrainGenericRegister(I.getOperand(2).getReg(), |
| AArch64::GPR32RegClass, MRI); |
| I.getOperand(2).setReg(SrcReg); |
| |
| return constrainSelectedInstRegOperands(I, TII, TRI, RBI); |
| } |
| case TargetOpcode::G_FRAME_INDEX: { |
| // allocas and G_FRAME_INDEX are only supported in addrspace(0). |
| if (Ty != LLT::pointer(0, 64)) { |
| LLVM_DEBUG(dbgs() << "G_FRAME_INDEX pointer has type: " << Ty |
| << ", expected: " << LLT::pointer(0, 64) << '\n'); |
| return false; |
| } |
| I.setDesc(TII.get(AArch64::ADDXri)); |
| |
| // MOs for a #0 shifted immediate. |
| I.addOperand(MachineOperand::CreateImm(0)); |
| I.addOperand(MachineOperand::CreateImm(0)); |
| |
| return constrainSelectedInstRegOperands(I, TII, TRI, RBI); |
| } |
| |
| case TargetOpcode::G_GLOBAL_VALUE: { |
| auto GV = I.getOperand(1).getGlobal(); |
| if (GV->isThreadLocal()) |
| return selectTLSGlobalValue(I, MRI); |
| |
| unsigned OpFlags = STI.ClassifyGlobalReference(GV, TM); |
| if (OpFlags & AArch64II::MO_GOT) { |
| I.setDesc(TII.get(AArch64::LOADgot)); |
| I.getOperand(1).setTargetFlags(OpFlags); |
| } else if (TM.getCodeModel() == CodeModel::Large) { |
| // Materialize the global using movz/movk instructions. |
| materializeLargeCMVal(I, GV, OpFlags); |
| I.eraseFromParent(); |
| return true; |
| } else if (TM.getCodeModel() == CodeModel::Tiny) { |
| I.setDesc(TII.get(AArch64::ADR)); |
| I.getOperand(1).setTargetFlags(OpFlags); |
| } else { |
| I.setDesc(TII.get(AArch64::MOVaddr)); |
| I.getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_PAGE); |
| MachineInstrBuilder MIB(MF, I); |
| MIB.addGlobalAddress(GV, I.getOperand(1).getOffset(), |
| OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC); |
| } |
| return constrainSelectedInstRegOperands(I, TII, TRI, RBI); |
| } |
| |
| case TargetOpcode::G_ZEXTLOAD: |
| case TargetOpcode::G_LOAD: |
| case TargetOpcode::G_STORE: { |
| bool IsZExtLoad = I.getOpcode() == TargetOpcode::G_ZEXTLOAD; |
| MachineIRBuilder MIB(I); |
| |
| LLT PtrTy = MRI.getType(I.getOperand(1).getReg()); |
| |
| if (PtrTy != LLT::pointer(0, 64)) { |
| LLVM_DEBUG(dbgs() << "Load/Store pointer has type: " << PtrTy |
| << ", expected: " << LLT::pointer(0, 64) << '\n'); |
| return false; |
| } |
| |
| auto &MemOp = **I.memoperands_begin(); |
| if (MemOp.isAtomic()) { |
| // For now we just support s8 acquire loads to be able to compile stack |
| // protector code. |
| if (MemOp.getOrdering() == AtomicOrdering::Acquire && |
| MemOp.getSize() == 1) { |
| I.setDesc(TII.get(AArch64::LDARB)); |
| return constrainSelectedInstRegOperands(I, TII, TRI, RBI); |
| } |
| LLVM_DEBUG(dbgs() << "Atomic load/store not fully supported yet\n"); |
| return false; |
| } |
| unsigned MemSizeInBits = MemOp.getSize() * 8; |
| |
| const Register PtrReg = I.getOperand(1).getReg(); |
| #ifndef NDEBUG |
| const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, MRI, TRI); |
| // Sanity-check the pointer register. |
| assert(PtrRB.getID() == AArch64::GPRRegBankID && |
| "Load/Store pointer operand isn't a GPR"); |
| assert(MRI.getType(PtrReg).isPointer() && |
| "Load/Store pointer operand isn't a pointer"); |
| #endif |
| |
| const Register ValReg = I.getOperand(0).getReg(); |
| const RegisterBank &RB = *RBI.getRegBank(ValReg, MRI, TRI); |
| |
| const unsigned NewOpc = |
| selectLoadStoreUIOp(I.getOpcode(), RB.getID(), MemSizeInBits); |
| if (NewOpc == I.getOpcode()) |
| return false; |
| |
| I.setDesc(TII.get(NewOpc)); |
| |
| uint64_t Offset = 0; |
| auto *PtrMI = MRI.getVRegDef(PtrReg); |
| |
| // Try to fold a GEP into our unsigned immediate addressing mode. |
| if (PtrMI->getOpcode() == TargetOpcode::G_PTR_ADD) { |
| if (auto COff = getConstantVRegVal(PtrMI->getOperand(2).getReg(), MRI)) { |
| int64_t Imm = *COff; |
| const unsigned Size = MemSizeInBits / 8; |
| const unsigned Scale = Log2_32(Size); |
| if ((Imm & (Size - 1)) == 0 && Imm >= 0 && Imm < (0x1000 << Scale)) { |
| Register Ptr2Reg = PtrMI->getOperand(1).getReg(); |
| I.getOperand(1).setReg(Ptr2Reg); |
| PtrMI = MRI.getVRegDef(Ptr2Reg); |
| Offset = Imm / Size; |
| } |
| } |
| } |
| |
| // If we haven't folded anything into our addressing mode yet, try to fold |
| // a frame index into the base+offset. |
| if (!Offset && PtrMI->getOpcode() == TargetOpcode::G_FRAME_INDEX) |
| I.getOperand(1).ChangeToFrameIndex(PtrMI->getOperand(1).getIndex()); |
| |
| I.addOperand(MachineOperand::CreateImm(Offset)); |
| |
| // If we're storing a 0, use WZR/XZR. |
| if (auto CVal = getConstantVRegVal(ValReg, MRI)) { |
| if (*CVal == 0 && Opcode == TargetOpcode::G_STORE) { |
| if (I.getOpcode() == AArch64::STRWui) |
| I.getOperand(0).setReg(AArch64::WZR); |
| else if (I.getOpcode() == AArch64::STRXui) |
| I.getOperand(0).setReg(AArch64::XZR); |
| } |
| } |
| |
| if (IsZExtLoad) { |
| // The zextload from a smaller type to i32 should be handled by the importer. |
| if (MRI.getType(ValReg).getSizeInBits() != 64) |
| return false; |
| // If we have a ZEXTLOAD then change the load's type to be a narrower reg |
| //and zero_extend with SUBREG_TO_REG. |
| Register LdReg = MRI.createVirtualRegister(&AArch64::GPR32RegClass); |
| Register DstReg = I.getOperand(0).getReg(); |
| I.getOperand(0).setReg(LdReg); |
| |
| MIB.setInsertPt(MIB.getMBB(), std::next(I.getIterator())); |
| MIB.buildInstr(AArch64::SUBREG_TO_REG, {DstReg}, {}) |
| .addImm(0) |
| .addUse(LdReg) |
| .addImm(AArch64::sub_32); |
| constrainSelectedInstRegOperands(I, TII, TRI, RBI); |
| return RBI.constrainGenericRegister(DstReg, AArch64::GPR64allRegClass, |
| MRI); |
| } |
| return constrainSelectedInstRegOperands(I, TII, TRI, RBI); |
| } |
| |
| case TargetOpcode::G_SMULH: |
| case TargetOpcode::G_UMULH: { |
| // Reject the various things we don't support yet. |
| if (unsupportedBinOp(I, RBI, MRI, TRI)) |
| return false; |
| |
| const Register DefReg = I.getOperand(0).getReg(); |
| const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI); |
| |
| if (RB.getID() != AArch64::GPRRegBankID) { |
| LLVM_DEBUG(dbgs() << "G_[SU]MULH on bank: " << RB << ", expected: GPR\n"); |
| return false; |
| } |
| |
| if (Ty != LLT::scalar(64)) { |
| LLVM_DEBUG(dbgs() << "G_[SU]MULH has type: " << Ty |
| << ", expected: " << LLT::scalar(64) << '\n'); |
| return false; |
| } |
| |
| unsigned NewOpc = I.getOpcode() == TargetOpcode::G_SMULH ? AArch64::SMULHrr |
| : AArch64::UMULHrr; |
| I.setDesc(TII.get(NewOpc)); |
| |
| // Now that we selected an opcode, we need to constrain the register |
| // operands to use appropriate classes. |
| return constrainSelectedInstRegOperands(I, TII, TRI, RBI); |
| } |
| case TargetOpcode::G_FADD: |
| case TargetOpcode::G_FSUB: |
| case TargetOpcode::G_FMUL: |
| case TargetOpcode::G_FDIV: |
| |
| case TargetOpcode::G_ASHR: |
| if (MRI.getType(I.getOperand(0).getReg()).isVector()) |
| return selectVectorASHR(I, MRI); |
| LLVM_FALLTHROUGH; |
| case TargetOpcode::G_SHL: |
| if (Opcode == TargetOpcode::G_SHL && |
| MRI.getType(I.getOperand(0).getReg()).isVector()) |
| return selectVectorSHL(I, MRI); |
| LLVM_FALLTHROUGH; |
| case TargetOpcode::G_OR: |
| case TargetOpcode::G_LSHR: { |
| // Reject the various things we don't support yet. |
| if (unsupportedBinOp(I, RBI, MRI, TRI)) |
| return false; |
| |
| const unsigned OpSize = Ty.getSizeInBits(); |
| |
| const Register DefReg = I.getOperand(0).getReg(); |
| const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI); |
| |
| const unsigned NewOpc = selectBinaryOp(I.getOpcode(), RB.getID(), OpSize); |
| if (NewOpc == I.getOpcode()) |
| return false; |
| |
| I.setDesc(TII.get(NewOpc)); |
| // FIXME: Should the type be always reset in setDesc? |
| |
| // Now that we selected an opcode, we need to constrain the register |
| // operands to use appropriate classes. |
| return constrainSelectedInstRegOperands(I, TII, TRI, RBI); |
| } |
| |
| case TargetOpcode::G_PTR_ADD: { |
| MachineIRBuilder MIRBuilder(I); |
| emitADD(I.getOperand(0).getReg(), I.getOperand(1), I.getOperand(2), |
| MIRBuilder); |
| I.eraseFromParent(); |
| return true; |
| } |
| case TargetOpcode::G_UADDO: { |
| // TODO: Support other types. |
| unsigned OpSize = Ty.getSizeInBits(); |
| if (OpSize != 32 && OpSize != 64) { |
| LLVM_DEBUG( |
| dbgs() |
| << "G_UADDO currently only supported for 32 and 64 b types.\n"); |
| return false; |
| } |
| |
| // TODO: Support vectors. |
| if (Ty.isVector()) { |
| LLVM_DEBUG(dbgs() << "G_UADDO currently only supported for scalars.\n"); |
| return false; |
| } |
| |
| // Add and set the set condition flag. |
| unsigned AddsOpc = OpSize == 32 ? AArch64::ADDSWrr : AArch64::ADDSXrr; |
| MachineIRBuilder MIRBuilder(I); |
| auto AddsMI = MIRBuilder.buildInstr( |
| AddsOpc, {I.getOperand(0).getReg()}, |
| {I.getOperand(2).getReg(), I.getOperand(3).getReg()}); |
| constrainSelectedInstRegOperands(*AddsMI, TII, TRI, RBI); |
| |
| // Now, put the overflow result in the register given by the first operand |
| // to the G_UADDO. CSINC increments the result when the predicate is false, |
| // so to get the increment when it's true, we need to use the inverse. In |
| // this case, we want to increment when carry is set. |
| auto CsetMI = MIRBuilder |
| .buildInstr(AArch64::CSINCWr, {I.getOperand(1).getReg()}, |
| {Register(AArch64::WZR), Register(AArch64::WZR)}) |
| .addImm(getInvertedCondCode(AArch64CC::HS)); |
| constrainSelectedInstRegOperands(*CsetMI, TII, TRI, RBI); |
| I.eraseFromParent(); |
| return true; |
| } |
| |
| case TargetOpcode::G_PTR_MASK: { |
| uint64_t Align = I.getOperand(2).getImm(); |
| if (Align >= 64 || Align == 0) |
| return false; |
| |
| uint64_t Mask = ~((1ULL << Align) - 1); |
| I.setDesc(TII.get(AArch64::ANDXri)); |
| I.getOperand(2).setImm(AArch64_AM::encodeLogicalImmediate(Mask, 64)); |
| |
| return constrainSelectedInstRegOperands(I, TII, TRI, RBI); |
| } |
| case TargetOpcode::G_PTRTOINT: |
| case TargetOpcode::G_TRUNC: { |
| const LLT DstTy = MRI.getType(I.getOperand(0).getReg()); |
| const LLT SrcTy = MRI.getType(I.getOperand(1).getReg()); |
| |
| const Register DstReg = I.getOperand(0).getReg(); |
| const Register SrcReg = I.getOperand(1).getReg(); |
| |
| const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI); |
| const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI); |
| |
| if (DstRB.getID() != SrcRB.getID()) { |
| LLVM_DEBUG( |
| dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n"); |
| return false; |
| } |
| |
| if (DstRB.getID() == AArch64::GPRRegBankID) { |
| const TargetRegisterClass *DstRC = |
| getRegClassForTypeOnBank(DstTy, DstRB, RBI); |
| if (!DstRC) |
| return false; |
| |
| const TargetRegisterClass *SrcRC = |
| getRegClassForTypeOnBank(SrcTy, SrcRB, RBI); |
| if (!SrcRC) |
| return false; |
| |
| if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) || |
| !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { |
| LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC/G_PTRTOINT\n"); |
| return false; |
| } |
| |
| if (DstRC == SrcRC) { |
| // Nothing to be done |
| } else if (Opcode == TargetOpcode::G_TRUNC && DstTy == LLT::scalar(32) && |
| SrcTy == LLT::scalar(64)) { |
| llvm_unreachable("TableGen can import this case"); |
| return false; |
| } else if (DstRC == &AArch64::GPR32RegClass && |
| SrcRC == &AArch64::GPR64RegClass) { |
| I.getOperand(1).setSubReg(AArch64::sub_32); |
| } else { |
| LLVM_DEBUG( |
| dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n"); |
| return false; |
| } |
| |
| I.setDesc(TII.get(TargetOpcode::COPY)); |
| return true; |
| } else if (DstRB.getID() == AArch64::FPRRegBankID) { |
| if (DstTy == LLT::vector(4, 16) && SrcTy == LLT::vector(4, 32)) { |
| I.setDesc(TII.get(AArch64::XTNv4i16)); |
| constrainSelectedInstRegOperands(I, TII, TRI, RBI); |
| return true; |
| } |
| |
| if (!SrcTy.isVector() && SrcTy.getSizeInBits() == 128) { |
| MachineIRBuilder MIB(I); |
| MachineInstr *Extract = emitExtractVectorElt( |
| DstReg, DstRB, LLT::scalar(DstTy.getSizeInBits()), SrcReg, 0, MIB); |
| if (!Extract) |
| return false; |
| I.eraseFromParent(); |
| return true; |
| } |
| } |
| |
| return false; |
| } |
| |
| case TargetOpcode::G_ANYEXT: { |
| const Register DstReg = I.getOperand(0).getReg(); |
| const Register SrcReg = I.getOperand(1).getReg(); |
| |
| const RegisterBank &RBDst = *RBI.getRegBank(DstReg, MRI, TRI); |
| if (RBDst.getID() != AArch64::GPRRegBankID) { |
| LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBDst |
| << ", expected: GPR\n"); |
| return false; |
| } |
| |
| const RegisterBank &RBSrc = *RBI.getRegBank(SrcReg, MRI, TRI); |
| if (RBSrc.getID() != AArch64::GPRRegBankID) { |
| LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBSrc |
| << ", expected: GPR\n"); |
| return false; |
| } |
| |
| const unsigned DstSize = MRI.getType(DstReg).getSizeInBits(); |
| |
| if (DstSize == 0) { |
| LLVM_DEBUG(dbgs() << "G_ANYEXT operand has no size, not a gvreg?\n"); |
| return false; |
| } |
| |
| if (DstSize != 64 && DstSize > 32) { |
| LLVM_DEBUG(dbgs() << "G_ANYEXT to size: " << DstSize |
| << ", expected: 32 or 64\n"); |
| return false; |
| } |
| // At this point G_ANYEXT is just like a plain COPY, but we need |
| // to explicitly form the 64-bit value if any. |
| if (DstSize > 32) { |
| Register ExtSrc = MRI.createVirtualRegister(&AArch64::GPR64allRegClass); |
| BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::SUBREG_TO_REG)) |
| .addDef(ExtSrc) |
| .addImm(0) |
| .addUse(SrcReg) |
| .addImm(AArch64::sub_32); |
| I.getOperand(1).setReg(ExtSrc); |
| } |
| return selectCopy(I, TII, MRI, TRI, RBI); |
| } |
| |
| case TargetOpcode::G_ZEXT: |
| case TargetOpcode::G_SEXT: { |
| unsigned Opcode = I.getOpcode(); |
| const bool IsSigned = Opcode == TargetOpcode::G_SEXT; |
| const Register DefReg = I.getOperand(0).getReg(); |
| const Register SrcReg = I.getOperand(1).getReg(); |
| const LLT DstTy = MRI.getType(DefReg); |
| const LLT SrcTy = MRI.getType(SrcReg); |
| unsigned DstSize = DstTy.getSizeInBits(); |
| unsigned SrcSize = SrcTy.getSizeInBits(); |
| |
| if (DstTy.isVector()) |
| return false; // Should be handled by imported patterns. |
| |
| assert((*RBI.getRegBank(DefReg, MRI, TRI)).getID() == |
| AArch64::GPRRegBankID && |
| "Unexpected ext regbank"); |
| |
| MachineIRBuilder MIB(I); |
| MachineInstr *ExtI; |
| |
| // First check if we're extending the result of a load which has a dest type |
| // smaller than 32 bits, then this zext is redundant. GPR32 is the smallest |
| // GPR register on AArch64 and all loads which are smaller automatically |
| // zero-extend the upper bits. E.g. |
| // %v(s8) = G_LOAD %p, :: (load 1) |
| // %v2(s32) = G_ZEXT %v(s8) |
| if (!IsSigned) { |
| auto *LoadMI = getOpcodeDef(TargetOpcode::G_LOAD, SrcReg, MRI); |
| if (LoadMI && |
| RBI.getRegBank(SrcReg, MRI, TRI)->getID() == AArch64::GPRRegBankID) { |
| const MachineMemOperand *MemOp = *LoadMI->memoperands_begin(); |
| unsigned BytesLoaded = MemOp->getSize(); |
| if (BytesLoaded < 4 && SrcTy.getSizeInBytes() == BytesLoaded) |
| return selectCopy(I, TII, MRI, TRI, RBI); |
| } |
| } |
| |
| if (DstSize == 64) { |
| // FIXME: Can we avoid manually doing this? |
| if (!RBI.constrainGenericRegister(SrcReg, AArch64::GPR32RegClass, MRI)) { |
| LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(Opcode) |
| << " operand\n"); |
| return false; |
| } |
| |
| auto SubregToReg = |
| MIB.buildInstr(AArch64::SUBREG_TO_REG, {&AArch64::GPR64RegClass}, {}) |
| .addImm(0) |
| .addUse(SrcReg) |
| .addImm(AArch64::sub_32); |
| |
| ExtI = MIB.buildInstr(IsSigned ? AArch64::SBFMXri : AArch64::UBFMXri, |
| {DefReg}, {SubregToReg}) |
| .addImm(0) |
| .addImm(SrcSize - 1); |
| } else if (DstSize <= 32) { |
| ExtI = MIB.buildInstr(IsSigned ? AArch64::SBFMWri : AArch64::UBFMWri, |
| {DefReg}, {SrcReg}) |
| .addImm(0) |
| .addImm(SrcSize - 1); |
| } else { |
| return false; |
| } |
| |
| constrainSelectedInstRegOperands(*ExtI, TII, TRI, RBI); |
| I.eraseFromParent(); |
| return true; |
| } |
| |
| case TargetOpcode::G_SITOFP: |
| case TargetOpcode::G_UITOFP: |
| case TargetOpcode::G_FPTOSI: |
| case TargetOpcode::G_FPTOUI: { |
| const LLT DstTy = MRI.getType(I.getOperand(0).getReg()), |
| SrcTy = MRI.getType(I.getOperand(1).getReg()); |
| const unsigned NewOpc = selectFPConvOpc(Opcode, DstTy, SrcTy); |
| if (NewOpc == Opcode) |
| return false; |
| |
| I.setDesc(TII.get(NewOpc)); |
| constrainSelectedInstRegOperands(I, TII, TRI, RBI); |
| |
| return true; |
| } |
| |
| |
| case TargetOpcode::G_INTTOPTR: |
| // The importer is currently unable to import pointer types since they |
| // didn't exist in SelectionDAG. |
| return selectCopy(I, TII, MRI, TRI, RBI); |
| |
| case TargetOpcode::G_BITCAST: |
| // Imported SelectionDAG rules can handle every bitcast except those that |
| // bitcast from a type to the same type. Ideally, these shouldn't occur |
| // but we might not run an optimizer that deletes them. The other exception |
| // is bitcasts involving pointer types, as SelectionDAG has no knowledge |
| // of them. |
| return selectCopy(I, TII, MRI, TRI, RBI); |
| |
| case TargetOpcode::G_SELECT: { |
| if (MRI.getType(I.getOperand(1).getReg()) != LLT::scalar(1)) { |
| LLVM_DEBUG(dbgs() << "G_SELECT cond has type: " << Ty |
| << ", expected: " << LLT::scalar(1) << '\n'); |
| return false; |
| } |
| |
| const Register CondReg = I.getOperand(1).getReg(); |
| const Register TReg = I.getOperand(2).getReg(); |
| const Register FReg = I.getOperand(3).getReg(); |
| |
| if (tryOptSelect(I)) |
| return true; |
| |
| Register CSelOpc = selectSelectOpc(I, MRI, RBI); |
| MachineInstr &TstMI = |
| *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::ANDSWri)) |
| .addDef(AArch64::WZR) |
| .addUse(CondReg) |
| .addImm(AArch64_AM::encodeLogicalImmediate(1, 32)); |
| |
| MachineInstr &CSelMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CSelOpc)) |
| .addDef(I.getOperand(0).getReg()) |
| .addUse(TReg) |
| .addUse(FReg) |
| .addImm(AArch64CC::NE); |
| |
| constrainSelectedInstRegOperands(TstMI, TII, TRI, RBI); |
| constrainSelectedInstRegOperands(CSelMI, TII, TRI, RBI); |
| |
| I.eraseFromParent(); |
| return true; |
| } |
| case TargetOpcode::G_ICMP: { |
| if (Ty.isVector()) |
| return selectVectorICmp(I, MRI); |
| |
| if (Ty != LLT::scalar(32)) { |
| LLVM_DEBUG(dbgs() << "G_ICMP result has type: " << Ty |
| << ", expected: " << LLT::scalar(32) << '\n'); |
| return false; |
| } |
| |
| MachineIRBuilder MIRBuilder(I); |
| if (!emitIntegerCompare(I.getOperand(2), I.getOperand(3), I.getOperand(1), |
| MIRBuilder)) |
| return false; |
| emitCSetForICMP(I.getOperand(0).getReg(), I.getOperand(1).getPredicate(), |
| MIRBuilder); |
| I.eraseFromParent(); |
| return true; |
| } |
| |
| case TargetOpcode::G_FCMP: { |
| if (Ty != LLT::scalar(32)) { |
| LLVM_DEBUG(dbgs() << "G_FCMP result has type: " << Ty |
| << ", expected: " << LLT::scalar(32) << '\n'); |
| return false; |
| } |
| |
| unsigned CmpOpc = selectFCMPOpc(I, MRI); |
| if (!CmpOpc) |
| return false; |
| |
| // FIXME: regbank |
| |
| AArch64CC::CondCode CC1, CC2; |
| changeFCMPPredToAArch64CC( |
| (CmpInst::Predicate)I.getOperand(1).getPredicate(), CC1, CC2); |
| |
| // Partially build the compare. Decide if we need to add a use for the |
| // third operand based off whether or not we're comparing against 0.0. |
| auto CmpMI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(CmpOpc)) |
| .addUse(I.getOperand(2).getReg()); |
| |
| // If we don't have an immediate compare, then we need to add a use of the |
| // register which wasn't used for the immediate. |
| // Note that the immediate will always be the last operand. |
| if (CmpOpc != AArch64::FCMPSri && CmpOpc != AArch64::FCMPDri) |
| CmpMI = CmpMI.addUse(I.getOperand(3).getReg()); |
| |
| const Register DefReg = I.getOperand(0).getReg(); |
| Register Def1Reg = DefReg; |
| if (CC2 != AArch64CC::AL) |
| Def1Reg = MRI.createVirtualRegister(&AArch64::GPR32RegClass); |
| |
| MachineInstr &CSetMI = |
| *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr)) |
| .addDef(Def1Reg) |
| .addUse(AArch64::WZR) |
| .addUse(AArch64::WZR) |
| .addImm(getInvertedCondCode(CC1)); |
| |
| if (CC2 != AArch64CC::AL) { |
| Register Def2Reg = MRI.createVirtualRegister(&AArch64::GPR32RegClass); |
| MachineInstr &CSet2MI = |
| *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr)) |
| .addDef(Def2Reg) |
| .addUse(AArch64::WZR) |
| .addUse(AArch64::WZR) |
| .addImm(getInvertedCondCode(CC2)); |
| MachineInstr &OrMI = |
| *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::ORRWrr)) |
| .addDef(DefReg) |
| .addUse(Def1Reg) |
| .addUse(Def2Reg); |
| constrainSelectedInstRegOperands(OrMI, TII, TRI, RBI); |
| constrainSelectedInstRegOperands(CSet2MI, TII, TRI, RBI); |
| } |
| constrainSelectedInstRegOperands(*CmpMI, TII, TRI, RBI); |
| constrainSelectedInstRegOperands(CSetMI, TII, TRI, RBI); |
| |
| I.eraseFromParent(); |
| return true; |
| } |
| case TargetOpcode::G_VASTART: |
| return STI.isTargetDarwin() ? selectVaStartDarwin(I, MF, MRI) |
| : selectVaStartAAPCS(I, MF, MRI); |
| case TargetOpcode::G_INTRINSIC: |
| return selectIntrinsic(I, MRI); |
| case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: |
| return selectIntrinsicWithSideEffects(I, MRI); |
| case TargetOpcode::G_IMPLICIT_DEF: { |
| I.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF)); |
| const LLT DstTy = MRI.getType(I.getOperand(0).getReg()); |
| const Register DstReg = I.getOperand(0).getReg(); |
| const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI); |
| const TargetRegisterClass *DstRC = |
| getRegClassForTypeOnBank(DstTy, DstRB, RBI); |
| RBI.constrainGenericRegister(DstReg, *DstRC, MRI); |
| return true; |
| } |
| case TargetOpcode::G_BLOCK_ADDR: { |
| if (TM.getCodeModel() == CodeModel::Large) { |
| materializeLargeCMVal(I, I.getOperand(1).getBlockAddress(), 0); |
| I.eraseFromParent(); |
| return true; |
| } else { |
| I.setDesc(TII.get(AArch64::MOVaddrBA)); |
| auto MovMI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::MOVaddrBA), |
| I.getOperand(0).getReg()) |
| .addBlockAddress(I.getOperand(1).getBlockAddress(), |
| /* Offset */ 0, AArch64II::MO_PAGE) |
| .addBlockAddress( |
| I.getOperand(1).getBlockAddress(), /* Offset */ 0, |
| AArch64II::MO_NC | AArch64II::MO_PAGEOFF); |
| I.eraseFromParent(); |
| return constrainSelectedInstRegOperands(*MovMI, TII, TRI, RBI); |
| } |
| } |
| case TargetOpcode::G_INTRINSIC_TRUNC: |
| return selectIntrinsicTrunc(I, MRI); |
| case TargetOpcode::G_INTRINSIC_ROUND: |
| return selectIntrinsicRound(I, MRI); |
| case TargetOpcode::G_BUILD_VECTOR: |
| return selectBuildVector(I, MRI); |
| case TargetOpcode::G_MERGE_VALUES: |
| return selectMergeValues(I, MRI); |
| case TargetOpcode::G_UNMERGE_VALUES: |
| return selectUnmergeValues(I, MRI); |
| case TargetOpcode::G_SHUFFLE_VECTOR: |
| return selectShuffleVector(I, MRI); |
| case TargetOpcode::G_EXTRACT_VECTOR_ELT: |
| return selectExtractElt(I, MRI); |
| case TargetOpcode::G_INSERT_VECTOR_ELT: |
| return selectInsertElt(I, MRI); |
| case TargetOpcode::G_CONCAT_VECTORS: |
| return selectConcatVectors(I, MRI); |
| case TargetOpcode::G_JUMP_TABLE: |
| return selectJumpTable(I, MRI); |
| } |
| |
| return false; |
| } |
| |
| bool AArch64InstructionSelector::selectBrJT(MachineInstr &I, |
| MachineRegisterInfo &MRI) const { |
| assert(I.getOpcode() == TargetOpcode::G_BRJT && "Expected G_BRJT"); |
| Register JTAddr = I.getOperand(0).getReg(); |
| unsigned JTI = I.getOperand(1).getIndex(); |
| Register Index = I.getOperand(2).getReg(); |
| MachineIRBuilder MIB(I); |
| |
| Register TargetReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass); |
| Register ScratchReg = MRI.createVirtualRegister(&AArch64::GPR64spRegClass); |
| MIB.buildInstr(AArch64::JumpTableDest32, {TargetReg, ScratchReg}, |
| {JTAddr, Index}) |
| .addJumpTableIndex(JTI); |
| |
| // Build the indirect branch. |
| MIB.buildInstr(AArch64::BR, {}, {TargetReg}); |
| I.eraseFromParent(); |
| return true; |
| } |
| |
| bool AArch64InstructionSelector::selectJumpTable( |
| MachineInstr &I, MachineRegisterInfo &MRI) const { |
| assert(I.getOpcode() == TargetOpcode::G_JUMP_TABLE && "Expected jump table"); |
| assert(I.getOperand(1).isJTI() && "Jump table op should have a JTI!"); |
| |
| Register DstReg = I.getOperand(0).getReg(); |
| unsigned JTI = I.getOperand(1).getIndex(); |
| // We generate a MOVaddrJT which will get expanded to an ADRP + ADD later. |
| MachineIRBuilder MIB(I); |
| auto MovMI = |
| MIB.buildInstr(AArch64::MOVaddrJT, {DstReg}, {}) |
| .addJumpTableIndex(JTI, AArch64II::MO_PAGE) |
| .addJumpTableIndex(JTI, AArch64II::MO_NC | AArch64II::MO_PAGEOFF); |
| I.eraseFromParent(); |
| return constrainSelectedInstRegOperands(*MovMI, TII, TRI, RBI); |
| } |
| |
| bool AArch64InstructionSelector::selectTLSGlobalValue( |
| MachineInstr &I, MachineRegisterInfo &MRI) const { |
| if (!STI.isTargetMachO()) |
| return false; |
| MachineFunction &MF = *I.getParent()->getParent(); |
| MF.getFrameInfo().setAdjustsStack(true); |
| |
| const GlobalValue &GV = *I.getOperand(1).getGlobal(); |
| MachineIRBuilder MIB(I); |
| |
| MIB.buildInstr(AArch64::LOADgot, {AArch64::X0}, {}) |
| .addGlobalAddress(&GV, 0, AArch64II::MO_TLS); |
| |
| auto Load = MIB.buildInstr(AArch64::LDRXui, {&AArch64::GPR64commonRegClass}, |
| {Register(AArch64::X0)}) |
| .addImm(0); |
| |
| // TLS calls preserve all registers except those that absolutely must be |
| // trashed: X0 (it takes an argument), LR (it's a call) and NZCV (let's not be |
| // silly). |
| MIB.buildInstr(AArch64::BLR, {}, {Load}) |
| .addDef(AArch64::X0, RegState::Implicit) |
| .addRegMask(TRI.getTLSCallPreservedMask()); |
| |
| MIB.buildCopy(I.getOperand(0).getReg(), Register(AArch64::X0)); |
| RBI.constrainGenericRegister(I.getOperand(0).getReg(), AArch64::GPR64RegClass, |
| MRI); |
| I.eraseFromParent(); |
| return true; |
| } |
| |
| bool AArch64InstructionSelector::selectIntrinsicTrunc( |
| MachineInstr &I, MachineRegisterInfo &MRI) const { |
| const LLT SrcTy = MRI.getType(I.getOperand(0).getReg()); |
| |
| // Select the correct opcode. |
| unsigned Opc = 0; |
| if (!SrcTy.isVector()) { |
| switch (SrcTy.getSizeInBits()) { |
| default: |
| case 16: |
| Opc = AArch64::FRINTZHr; |
| break; |
| case 32: |
| Opc = AArch64::FRINTZSr; |
| break; |
| case 64: |
| Opc = AArch64::FRINTZDr; |
| break; |
| } |
| } else { |
| unsigned NumElts = SrcTy.getNumElements(); |
| switch (SrcTy.getElementType().getSizeInBits()) { |
| default: |
| break; |
| case 16: |
| if (NumElts == 4) |
| Opc = AArch64::FRINTZv4f16; |
| else if (NumElts == 8) |
| Opc = AArch64::FRINTZv8f16; |
| break; |
| case 32: |
| if (NumElts == 2) |
| Opc = AArch64::FRINTZv2f32; |
| else if (NumElts == 4) |
| Opc = AArch64::FRINTZv4f32; |
| break; |
| case 64: |
| if (NumElts == 2) |
| Opc = AArch64::FRINTZv2f64; |
| break; |
| } |
| } |
| |
| if (!Opc) { |
| // Didn't get an opcode above, bail. |
| LLVM_DEBUG(dbgs() << "Unsupported type for G_INTRINSIC_TRUNC!\n"); |
| return false; |
| } |
| |
| // Legalization would have set us up perfectly for this; we just need to |
| // set the opcode and move on. |
| I.setDesc(TII.get(Opc)); |
| return constrainSelectedInstRegOperands(I, TII, TRI, RBI); |
| } |
| |
| bool AArch64InstructionSelector::selectIntrinsicRound( |
| MachineInstr &I, MachineRegisterInfo &MRI) const { |
| const LLT SrcTy = MRI.getType(I.getOperand(0).getReg()); |
|