| //===- AArch64InstrInfo.cpp - AArch64 Instruction Information -------------===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| // |
| // This file contains the AArch64 implementation of the TargetInstrInfo class. |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "AArch64InstrInfo.h" |
| #include "AArch64MachineFunctionInfo.h" |
| #include "AArch64Subtarget.h" |
| #include "MCTargetDesc/AArch64AddressingModes.h" |
| #include "Utils/AArch64BaseInfo.h" |
| #include "llvm/ADT/ArrayRef.h" |
| #include "llvm/ADT/STLExtras.h" |
| #include "llvm/ADT/SmallVector.h" |
| #include "llvm/CodeGen/MachineBasicBlock.h" |
| #include "llvm/CodeGen/MachineFrameInfo.h" |
| #include "llvm/CodeGen/MachineFunction.h" |
| #include "llvm/CodeGen/MachineInstr.h" |
| #include "llvm/CodeGen/MachineInstrBuilder.h" |
| #include "llvm/CodeGen/MachineMemOperand.h" |
| #include "llvm/CodeGen/MachineOperand.h" |
| #include "llvm/CodeGen/MachineRegisterInfo.h" |
| #include "llvm/CodeGen/MachineModuleInfo.h" |
| #include "llvm/CodeGen/StackMaps.h" |
| #include "llvm/CodeGen/TargetRegisterInfo.h" |
| #include "llvm/CodeGen/TargetSubtargetInfo.h" |
| #include "llvm/IR/DebugInfoMetadata.h" |
| #include "llvm/IR/DebugLoc.h" |
| #include "llvm/IR/GlobalValue.h" |
| #include "llvm/MC/MCAsmInfo.h" |
| #include "llvm/MC/MCInst.h" |
| #include "llvm/MC/MCInstrDesc.h" |
| #include "llvm/Support/Casting.h" |
| #include "llvm/Support/CodeGen.h" |
| #include "llvm/Support/CommandLine.h" |
| #include "llvm/Support/Compiler.h" |
| #include "llvm/Support/ErrorHandling.h" |
| #include "llvm/Support/MathExtras.h" |
| #include "llvm/Target/TargetMachine.h" |
| #include "llvm/Target/TargetOptions.h" |
| #include <cassert> |
| #include <cstdint> |
| #include <iterator> |
| #include <utility> |
| |
| using namespace llvm; |
| |
| #define GET_INSTRINFO_CTOR_DTOR |
| #include "AArch64GenInstrInfo.inc" |
| |
| static cl::opt<unsigned> TBZDisplacementBits( |
| "aarch64-tbz-offset-bits", cl::Hidden, cl::init(14), |
| cl::desc("Restrict range of TB[N]Z instructions (DEBUG)")); |
| |
| static cl::opt<unsigned> CBZDisplacementBits( |
| "aarch64-cbz-offset-bits", cl::Hidden, cl::init(19), |
| cl::desc("Restrict range of CB[N]Z instructions (DEBUG)")); |
| |
| static cl::opt<unsigned> |
| BCCDisplacementBits("aarch64-bcc-offset-bits", cl::Hidden, cl::init(19), |
| cl::desc("Restrict range of Bcc instructions (DEBUG)")); |
| |
| AArch64InstrInfo::AArch64InstrInfo(const AArch64Subtarget &STI) |
| : AArch64GenInstrInfo(AArch64::ADJCALLSTACKDOWN, AArch64::ADJCALLSTACKUP, |
| AArch64::CATCHRET), |
| RI(STI.getTargetTriple()), Subtarget(STI) {} |
| |
| /// GetInstSize - Return the number of bytes of code the specified |
| /// instruction may be. This returns the maximum number of bytes. |
| unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { |
| const MachineBasicBlock &MBB = *MI.getParent(); |
| const MachineFunction *MF = MBB.getParent(); |
| const MCAsmInfo *MAI = MF->getTarget().getMCAsmInfo(); |
| |
| { |
| auto Op = MI.getOpcode(); |
| if (Op == AArch64::INLINEASM || Op == AArch64::INLINEASM_BR) |
| return getInlineAsmLength(MI.getOperand(0).getSymbolName(), *MAI); |
| } |
| |
| // Meta-instructions emit no code. |
| if (MI.isMetaInstruction()) |
| return 0; |
| |
| // FIXME: We currently only handle pseudoinstructions that don't get expanded |
| // before the assembly printer. |
| unsigned NumBytes = 0; |
| const MCInstrDesc &Desc = MI.getDesc(); |
| switch (Desc.getOpcode()) { |
| default: |
| // Anything not explicitly designated otherwise is a normal 4-byte insn. |
| NumBytes = 4; |
| break; |
| case TargetOpcode::STACKMAP: |
| // The upper bound for a stackmap intrinsic is the full length of its shadow |
| NumBytes = StackMapOpers(&MI).getNumPatchBytes(); |
| assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!"); |
| break; |
| case TargetOpcode::PATCHPOINT: |
| // The size of the patchpoint intrinsic is the number of bytes requested |
| NumBytes = PatchPointOpers(&MI).getNumPatchBytes(); |
| assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!"); |
| break; |
| case AArch64::TLSDESC_CALLSEQ: |
| // This gets lowered to an instruction sequence which takes 16 bytes |
| NumBytes = 16; |
| break; |
| case AArch64::JumpTableDest32: |
| case AArch64::JumpTableDest16: |
| case AArch64::JumpTableDest8: |
| NumBytes = 12; |
| break; |
| case AArch64::SPACE: |
| NumBytes = MI.getOperand(1).getImm(); |
| break; |
| } |
| |
| return NumBytes; |
| } |
| |
| static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target, |
| SmallVectorImpl<MachineOperand> &Cond) { |
| // Block ends with fall-through condbranch. |
| switch (LastInst->getOpcode()) { |
| default: |
| llvm_unreachable("Unknown branch instruction?"); |
| case AArch64::Bcc: |
| Target = LastInst->getOperand(1).getMBB(); |
| Cond.push_back(LastInst->getOperand(0)); |
| break; |
| case AArch64::CBZW: |
| case AArch64::CBZX: |
| case AArch64::CBNZW: |
| case AArch64::CBNZX: |
| Target = LastInst->getOperand(1).getMBB(); |
| Cond.push_back(MachineOperand::CreateImm(-1)); |
| Cond.push_back(MachineOperand::CreateImm(LastInst->getOpcode())); |
| Cond.push_back(LastInst->getOperand(0)); |
| break; |
| case AArch64::TBZW: |
| case AArch64::TBZX: |
| case AArch64::TBNZW: |
| case AArch64::TBNZX: |
| Target = LastInst->getOperand(2).getMBB(); |
| Cond.push_back(MachineOperand::CreateImm(-1)); |
| Cond.push_back(MachineOperand::CreateImm(LastInst->getOpcode())); |
| Cond.push_back(LastInst->getOperand(0)); |
| Cond.push_back(LastInst->getOperand(1)); |
| } |
| } |
| |
| static unsigned getBranchDisplacementBits(unsigned Opc) { |
| switch (Opc) { |
| default: |
| llvm_unreachable("unexpected opcode!"); |
| case AArch64::B: |
| return 64; |
| case AArch64::TBNZW: |
| case AArch64::TBZW: |
| case AArch64::TBNZX: |
| case AArch64::TBZX: |
| return TBZDisplacementBits; |
| case AArch64::CBNZW: |
| case AArch64::CBZW: |
| case AArch64::CBNZX: |
| case AArch64::CBZX: |
| return CBZDisplacementBits; |
| case AArch64::Bcc: |
| return BCCDisplacementBits; |
| } |
| } |
| |
| bool AArch64InstrInfo::isBranchOffsetInRange(unsigned BranchOp, |
| int64_t BrOffset) const { |
| unsigned Bits = getBranchDisplacementBits(BranchOp); |
| assert(Bits >= 3 && "max branch displacement must be enough to jump" |
| "over conditional branch expansion"); |
| return isIntN(Bits, BrOffset / 4); |
| } |
| |
| MachineBasicBlock * |
| AArch64InstrInfo::getBranchDestBlock(const MachineInstr &MI) const { |
| switch (MI.getOpcode()) { |
| default: |
| llvm_unreachable("unexpected opcode!"); |
| case AArch64::B: |
| return MI.getOperand(0).getMBB(); |
| case AArch64::TBZW: |
| case AArch64::TBNZW: |
| case AArch64::TBZX: |
| case AArch64::TBNZX: |
| return MI.getOperand(2).getMBB(); |
| case AArch64::CBZW: |
| case AArch64::CBNZW: |
| case AArch64::CBZX: |
| case AArch64::CBNZX: |
| case AArch64::Bcc: |
| return MI.getOperand(1).getMBB(); |
| } |
| } |
| |
| // Branch analysis. |
| bool AArch64InstrInfo::analyzeBranch(MachineBasicBlock &MBB, |
| MachineBasicBlock *&TBB, |
| MachineBasicBlock *&FBB, |
| SmallVectorImpl<MachineOperand> &Cond, |
| bool AllowModify) const { |
| // If the block has no terminators, it just falls into the block after it. |
| MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr(); |
| if (I == MBB.end()) |
| return false; |
| |
| if (!isUnpredicatedTerminator(*I)) |
| return false; |
| |
| // Get the last instruction in the block. |
| MachineInstr *LastInst = &*I; |
| |
| // If there is only one terminator instruction, process it. |
| unsigned LastOpc = LastInst->getOpcode(); |
| if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) { |
| if (isUncondBranchOpcode(LastOpc)) { |
| TBB = LastInst->getOperand(0).getMBB(); |
| return false; |
| } |
| if (isCondBranchOpcode(LastOpc)) { |
| // Block ends with fall-through condbranch. |
| parseCondBranch(LastInst, TBB, Cond); |
| return false; |
| } |
| return true; // Can't handle indirect branch. |
| } |
| |
| // Get the instruction before it if it is a terminator. |
| MachineInstr *SecondLastInst = &*I; |
| unsigned SecondLastOpc = SecondLastInst->getOpcode(); |
| |
| // If AllowModify is true and the block ends with two or more unconditional |
| // branches, delete all but the first unconditional branch. |
| if (AllowModify && isUncondBranchOpcode(LastOpc)) { |
| while (isUncondBranchOpcode(SecondLastOpc)) { |
| LastInst->eraseFromParent(); |
| LastInst = SecondLastInst; |
| LastOpc = LastInst->getOpcode(); |
| if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) { |
| // Return now the only terminator is an unconditional branch. |
| TBB = LastInst->getOperand(0).getMBB(); |
| return false; |
| } else { |
| SecondLastInst = &*I; |
| SecondLastOpc = SecondLastInst->getOpcode(); |
| } |
| } |
| } |
| |
| // If there are three terminators, we don't know what sort of block this is. |
| if (SecondLastInst && I != MBB.begin() && isUnpredicatedTerminator(*--I)) |
| return true; |
| |
| // If the block ends with a B and a Bcc, handle it. |
| if (isCondBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) { |
| parseCondBranch(SecondLastInst, TBB, Cond); |
| FBB = LastInst->getOperand(0).getMBB(); |
| return false; |
| } |
| |
| // If the block ends with two unconditional branches, handle it. The second |
| // one is not executed, so remove it. |
| if (isUncondBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) { |
| TBB = SecondLastInst->getOperand(0).getMBB(); |
| I = LastInst; |
| if (AllowModify) |
| I->eraseFromParent(); |
| return false; |
| } |
| |
| // ...likewise if it ends with an indirect branch followed by an unconditional |
| // branch. |
| if (isIndirectBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) { |
| I = LastInst; |
| if (AllowModify) |
| I->eraseFromParent(); |
| return true; |
| } |
| |
| // Otherwise, can't handle this. |
| return true; |
| } |
| |
| bool AArch64InstrInfo::reverseBranchCondition( |
| SmallVectorImpl<MachineOperand> &Cond) const { |
| if (Cond[0].getImm() != -1) { |
| // Regular Bcc |
| AArch64CC::CondCode CC = (AArch64CC::CondCode)(int)Cond[0].getImm(); |
| Cond[0].setImm(AArch64CC::getInvertedCondCode(CC)); |
| } else { |
| // Folded compare-and-branch |
| switch (Cond[1].getImm()) { |
| default: |
| llvm_unreachable("Unknown conditional branch!"); |
| case AArch64::CBZW: |
| Cond[1].setImm(AArch64::CBNZW); |
| break; |
| case AArch64::CBNZW: |
| Cond[1].setImm(AArch64::CBZW); |
| break; |
| case AArch64::CBZX: |
| Cond[1].setImm(AArch64::CBNZX); |
| break; |
| case AArch64::CBNZX: |
| Cond[1].setImm(AArch64::CBZX); |
| break; |
| case AArch64::TBZW: |
| Cond[1].setImm(AArch64::TBNZW); |
| break; |
| case AArch64::TBNZW: |
| Cond[1].setImm(AArch64::TBZW); |
| break; |
| case AArch64::TBZX: |
| Cond[1].setImm(AArch64::TBNZX); |
| break; |
| case AArch64::TBNZX: |
| Cond[1].setImm(AArch64::TBZX); |
| break; |
| } |
| } |
| |
| return false; |
| } |
| |
| unsigned AArch64InstrInfo::removeBranch(MachineBasicBlock &MBB, |
| int *BytesRemoved) const { |
| MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr(); |
| if (I == MBB.end()) |
| return 0; |
| |
| if (!isUncondBranchOpcode(I->getOpcode()) && |
| !isCondBranchOpcode(I->getOpcode())) |
| return 0; |
| |
| // Remove the branch. |
| I->eraseFromParent(); |
| |
| I = MBB.end(); |
| |
| if (I == MBB.begin()) { |
| if (BytesRemoved) |
| *BytesRemoved = 4; |
| return 1; |
| } |
| --I; |
| if (!isCondBranchOpcode(I->getOpcode())) { |
| if (BytesRemoved) |
| *BytesRemoved = 4; |
| return 1; |
| } |
| |
| // Remove the branch. |
| I->eraseFromParent(); |
| if (BytesRemoved) |
| *BytesRemoved = 8; |
| |
| return 2; |
| } |
| |
| void AArch64InstrInfo::instantiateCondBranch( |
| MachineBasicBlock &MBB, const DebugLoc &DL, MachineBasicBlock *TBB, |
| ArrayRef<MachineOperand> Cond) const { |
| if (Cond[0].getImm() != -1) { |
| // Regular Bcc |
| BuildMI(&MBB, DL, get(AArch64::Bcc)).addImm(Cond[0].getImm()).addMBB(TBB); |
| } else { |
| // Folded compare-and-branch |
| // Note that we use addOperand instead of addReg to keep the flags. |
| const MachineInstrBuilder MIB = |
| BuildMI(&MBB, DL, get(Cond[1].getImm())).add(Cond[2]); |
| if (Cond.size() > 3) |
| MIB.addImm(Cond[3].getImm()); |
| MIB.addMBB(TBB); |
| } |
| } |
| |
| unsigned AArch64InstrInfo::insertBranch( |
| MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, |
| ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const { |
| // Shouldn't be a fall through. |
| assert(TBB && "insertBranch must not be told to insert a fallthrough"); |
| |
| if (!FBB) { |
| if (Cond.empty()) // Unconditional branch? |
| BuildMI(&MBB, DL, get(AArch64::B)).addMBB(TBB); |
| else |
| instantiateCondBranch(MBB, DL, TBB, Cond); |
| |
| if (BytesAdded) |
| *BytesAdded = 4; |
| |
| return 1; |
| } |
| |
| // Two-way conditional branch. |
| instantiateCondBranch(MBB, DL, TBB, Cond); |
| BuildMI(&MBB, DL, get(AArch64::B)).addMBB(FBB); |
| |
| if (BytesAdded) |
| *BytesAdded = 8; |
| |
| return 2; |
| } |
| |
| // Find the original register that VReg is copied from. |
| static unsigned removeCopies(const MachineRegisterInfo &MRI, unsigned VReg) { |
| while (Register::isVirtualRegister(VReg)) { |
| const MachineInstr *DefMI = MRI.getVRegDef(VReg); |
| if (!DefMI->isFullCopy()) |
| return VReg; |
| VReg = DefMI->getOperand(1).getReg(); |
| } |
| return VReg; |
| } |
| |
| // Determine if VReg is defined by an instruction that can be folded into a |
| // csel instruction. If so, return the folded opcode, and the replacement |
| // register. |
| static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg, |
| unsigned *NewVReg = nullptr) { |
| VReg = removeCopies(MRI, VReg); |
| if (!Register::isVirtualRegister(VReg)) |
| return 0; |
| |
| bool Is64Bit = AArch64::GPR64allRegClass.hasSubClassEq(MRI.getRegClass(VReg)); |
| const MachineInstr *DefMI = MRI.getVRegDef(VReg); |
| unsigned Opc = 0; |
| unsigned SrcOpNum = 0; |
| switch (DefMI->getOpcode()) { |
| case AArch64::ADDSXri: |
| case AArch64::ADDSWri: |
| // if NZCV is used, do not fold. |
| if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, true) == -1) |
| return 0; |
| // fall-through to ADDXri and ADDWri. |
| LLVM_FALLTHROUGH; |
| case AArch64::ADDXri: |
| case AArch64::ADDWri: |
| // add x, 1 -> csinc. |
| if (!DefMI->getOperand(2).isImm() || DefMI->getOperand(2).getImm() != 1 || |
| DefMI->getOperand(3).getImm() != 0) |
| return 0; |
| SrcOpNum = 1; |
| Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr; |
| break; |
| |
| case AArch64::ORNXrr: |
| case AArch64::ORNWrr: { |
| // not x -> csinv, represented as orn dst, xzr, src. |
| unsigned ZReg = removeCopies(MRI, DefMI->getOperand(1).getReg()); |
| if (ZReg != AArch64::XZR && ZReg != AArch64::WZR) |
| return 0; |
| SrcOpNum = 2; |
| Opc = Is64Bit ? AArch64::CSINVXr : AArch64::CSINVWr; |
| break; |
| } |
| |
| case AArch64::SUBSXrr: |
| case AArch64::SUBSWrr: |
| // if NZCV is used, do not fold. |
| if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, true) == -1) |
| return 0; |
| // fall-through to SUBXrr and SUBWrr. |
| LLVM_FALLTHROUGH; |
| case AArch64::SUBXrr: |
| case AArch64::SUBWrr: { |
| // neg x -> csneg, represented as sub dst, xzr, src. |
| unsigned ZReg = removeCopies(MRI, DefMI->getOperand(1).getReg()); |
| if (ZReg != AArch64::XZR && ZReg != AArch64::WZR) |
| return 0; |
| SrcOpNum = 2; |
| Opc = Is64Bit ? AArch64::CSNEGXr : AArch64::CSNEGWr; |
| break; |
| } |
| default: |
| return 0; |
| } |
| assert(Opc && SrcOpNum && "Missing parameters"); |
| |
| if (NewVReg) |
| *NewVReg = DefMI->getOperand(SrcOpNum).getReg(); |
| return Opc; |
| } |
| |
| bool AArch64InstrInfo::canInsertSelect(const MachineBasicBlock &MBB, |
| ArrayRef<MachineOperand> Cond, |
| unsigned TrueReg, unsigned FalseReg, |
| int &CondCycles, int &TrueCycles, |
| int &FalseCycles) const { |
| // Check register classes. |
| const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); |
| const TargetRegisterClass *RC = |
| RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg)); |
| if (!RC) |
| return false; |
| |
| // Expanding cbz/tbz requires an extra cycle of latency on the condition. |
| unsigned ExtraCondLat = Cond.size() != 1; |
| |
| // GPRs are handled by csel. |
| // FIXME: Fold in x+1, -x, and ~x when applicable. |
| if (AArch64::GPR64allRegClass.hasSubClassEq(RC) || |
| AArch64::GPR32allRegClass.hasSubClassEq(RC)) { |
| // Single-cycle csel, csinc, csinv, and csneg. |
| CondCycles = 1 + ExtraCondLat; |
| TrueCycles = FalseCycles = 1; |
| if (canFoldIntoCSel(MRI, TrueReg)) |
| TrueCycles = 0; |
| else if (canFoldIntoCSel(MRI, FalseReg)) |
| FalseCycles = 0; |
| return true; |
| } |
| |
| // Scalar floating point is handled by fcsel. |
| // FIXME: Form fabs, fmin, and fmax when applicable. |
| if (AArch64::FPR64RegClass.hasSubClassEq(RC) || |
| AArch64::FPR32RegClass.hasSubClassEq(RC)) { |
| CondCycles = 5 + ExtraCondLat; |
| TrueCycles = FalseCycles = 2; |
| return true; |
| } |
| |
| // Can't do vectors. |
| return false; |
| } |
| |
| void AArch64InstrInfo::insertSelect(MachineBasicBlock &MBB, |
| MachineBasicBlock::iterator I, |
| const DebugLoc &DL, unsigned DstReg, |
| ArrayRef<MachineOperand> Cond, |
| unsigned TrueReg, unsigned FalseReg) const { |
| MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); |
| |
| // Parse the condition code, see parseCondBranch() above. |
| AArch64CC::CondCode CC; |
| switch (Cond.size()) { |
| default: |
| llvm_unreachable("Unknown condition opcode in Cond"); |
| case 1: // b.cc |
| CC = AArch64CC::CondCode(Cond[0].getImm()); |
| break; |
| case 3: { // cbz/cbnz |
| // We must insert a compare against 0. |
| bool Is64Bit; |
| switch (Cond[1].getImm()) { |
| default: |
| llvm_unreachable("Unknown branch opcode in Cond"); |
| case AArch64::CBZW: |
| Is64Bit = false; |
| CC = AArch64CC::EQ; |
| break; |
| case AArch64::CBZX: |
| Is64Bit = true; |
| CC = AArch64CC::EQ; |
| break; |
| case AArch64::CBNZW: |
| Is64Bit = false; |
| CC = AArch64CC::NE; |
| break; |
| case AArch64::CBNZX: |
| Is64Bit = true; |
| CC = AArch64CC::NE; |
| break; |
| } |
| Register SrcReg = Cond[2].getReg(); |
| if (Is64Bit) { |
| // cmp reg, #0 is actually subs xzr, reg, #0. |
| MRI.constrainRegClass(SrcReg, &AArch64::GPR64spRegClass); |
| BuildMI(MBB, I, DL, get(AArch64::SUBSXri), AArch64::XZR) |
| .addReg(SrcReg) |
| .addImm(0) |
| .addImm(0); |
| } else { |
| MRI.constrainRegClass(SrcReg, &AArch64::GPR32spRegClass); |
| BuildMI(MBB, I, DL, get(AArch64::SUBSWri), AArch64::WZR) |
| .addReg(SrcReg) |
| .addImm(0) |
| .addImm(0); |
| } |
| break; |
| } |
| case 4: { // tbz/tbnz |
| // We must insert a tst instruction. |
| switch (Cond[1].getImm()) { |
| default: |
| llvm_unreachable("Unknown branch opcode in Cond"); |
| case AArch64::TBZW: |
| case AArch64::TBZX: |
| CC = AArch64CC::EQ; |
| break; |
| case AArch64::TBNZW: |
| case AArch64::TBNZX: |
| CC = AArch64CC::NE; |
| break; |
| } |
| // cmp reg, #foo is actually ands xzr, reg, #1<<foo. |
| if (Cond[1].getImm() == AArch64::TBZW || Cond[1].getImm() == AArch64::TBNZW) |
| BuildMI(MBB, I, DL, get(AArch64::ANDSWri), AArch64::WZR) |
| .addReg(Cond[2].getReg()) |
| .addImm( |
| AArch64_AM::encodeLogicalImmediate(1ull << Cond[3].getImm(), 32)); |
| else |
| BuildMI(MBB, I, DL, get(AArch64::ANDSXri), AArch64::XZR) |
| .addReg(Cond[2].getReg()) |
| .addImm( |
| AArch64_AM::encodeLogicalImmediate(1ull << Cond[3].getImm(), 64)); |
| break; |
| } |
| } |
| |
| unsigned Opc = 0; |
| const TargetRegisterClass *RC = nullptr; |
| bool TryFold = false; |
| if (MRI.constrainRegClass(DstReg, &AArch64::GPR64RegClass)) { |
| RC = &AArch64::GPR64RegClass; |
| Opc = AArch64::CSELXr; |
| TryFold = true; |
| } else if (MRI.constrainRegClass(DstReg, &AArch64::GPR32RegClass)) { |
| RC = &AArch64::GPR32RegClass; |
| Opc = AArch64::CSELWr; |
| TryFold = true; |
| } else if (MRI.constrainRegClass(DstReg, &AArch64::FPR64RegClass)) { |
| RC = &AArch64::FPR64RegClass; |
| Opc = AArch64::FCSELDrrr; |
| } else if (MRI.constrainRegClass(DstReg, &AArch64::FPR32RegClass)) { |
| RC = &AArch64::FPR32RegClass; |
| Opc = AArch64::FCSELSrrr; |
| } |
| assert(RC && "Unsupported regclass"); |
| |
| // Try folding simple instructions into the csel. |
| if (TryFold) { |
| unsigned NewVReg = 0; |
| unsigned FoldedOpc = canFoldIntoCSel(MRI, TrueReg, &NewVReg); |
| if (FoldedOpc) { |
| // The folded opcodes csinc, csinc and csneg apply the operation to |
| // FalseReg, so we need to invert the condition. |
| CC = AArch64CC::getInvertedCondCode(CC); |
| TrueReg = FalseReg; |
| } else |
| FoldedOpc = canFoldIntoCSel(MRI, FalseReg, &NewVReg); |
| |
| // Fold the operation. Leave any dead instructions for DCE to clean up. |
| if (FoldedOpc) { |
| FalseReg = NewVReg; |
| Opc = FoldedOpc; |
| // The extends the live range of NewVReg. |
| MRI.clearKillFlags(NewVReg); |
| } |
| } |
| |
| // Pull all virtual register into the appropriate class. |
| MRI.constrainRegClass(TrueReg, RC); |
| MRI.constrainRegClass(FalseReg, RC); |
| |
| // Insert the csel. |
| BuildMI(MBB, I, DL, get(Opc), DstReg) |
| .addReg(TrueReg) |
| .addReg(FalseReg) |
| .addImm(CC); |
| } |
| |
| /// Returns true if a MOVi32imm or MOVi64imm can be expanded to an ORRxx. |
| static bool canBeExpandedToORR(const MachineInstr &MI, unsigned BitSize) { |
| uint64_t Imm = MI.getOperand(1).getImm(); |
| uint64_t UImm = Imm << (64 - BitSize) >> (64 - BitSize); |
| uint64_t Encoding; |
| return AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding); |
| } |
| |
| // FIXME: this implementation should be micro-architecture dependent, so a |
| // micro-architecture target hook should be introduced here in future. |
| bool AArch64InstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const { |
| if (!Subtarget.hasCustomCheapAsMoveHandling()) |
| return MI.isAsCheapAsAMove(); |
| |
| const unsigned Opcode = MI.getOpcode(); |
| |
| // Firstly, check cases gated by features. |
| |
| if (Subtarget.hasZeroCycleZeroingFP()) { |
| if (Opcode == AArch64::FMOVH0 || |
| Opcode == AArch64::FMOVS0 || |
| Opcode == AArch64::FMOVD0) |
| return true; |
| } |
| |
| if (Subtarget.hasZeroCycleZeroingGP()) { |
| if (Opcode == TargetOpcode::COPY && |
| (MI.getOperand(1).getReg() == AArch64::WZR || |
| MI.getOperand(1).getReg() == AArch64::XZR)) |
| return true; |
| } |
| |
| // Secondly, check cases specific to sub-targets. |
| |
| if (Subtarget.hasExynosCheapAsMoveHandling()) { |
| if (isExynosCheapAsMove(MI)) |
| return true; |
| |
| return MI.isAsCheapAsAMove(); |
| } |
| |
| // Finally, check generic cases. |
| |
| switch (Opcode) { |
| default: |
| return false; |
| |
| // add/sub on register without shift |
| case AArch64::ADDWri: |
| case AArch64::ADDXri: |
| case AArch64::SUBWri: |
| case AArch64::SUBXri: |
| return (MI.getOperand(3).getImm() == 0); |
| |
| // logical ops on immediate |
| case AArch64::ANDWri: |
| case AArch64::ANDXri: |
| case AArch64::EORWri: |
| case AArch64::EORXri: |
| case AArch64::ORRWri: |
| case AArch64::ORRXri: |
| return true; |
| |
| // logical ops on register without shift |
| case AArch64::ANDWrr: |
| case AArch64::ANDXrr: |
| case AArch64::BICWrr: |
| case AArch64::BICXrr: |
| case AArch64::EONWrr: |
| case AArch64::EONXrr: |
| case AArch64::EORWrr: |
| case AArch64::EORXrr: |
| case AArch64::ORNWrr: |
| case AArch64::ORNXrr: |
| case AArch64::ORRWrr: |
| case AArch64::ORRXrr: |
| return true; |
| |
| // If MOVi32imm or MOVi64imm can be expanded into ORRWri or |
| // ORRXri, it is as cheap as MOV |
| case AArch64::MOVi32imm: |
| return canBeExpandedToORR(MI, 32); |
| case AArch64::MOVi64imm: |
| return canBeExpandedToORR(MI, 64); |
| } |
| |
| llvm_unreachable("Unknown opcode to check as cheap as a move!"); |
| } |
| |
| bool AArch64InstrInfo::isFalkorShiftExtFast(const MachineInstr &MI) { |
| switch (MI.getOpcode()) { |
| default: |
| return false; |
| |
| case AArch64::ADDWrs: |
| case AArch64::ADDXrs: |
| case AArch64::ADDSWrs: |
| case AArch64::ADDSXrs: { |
| unsigned Imm = MI.getOperand(3).getImm(); |
| unsigned ShiftVal = AArch64_AM::getShiftValue(Imm); |
| if (ShiftVal == 0) |
| return true; |
| return AArch64_AM::getShiftType(Imm) == AArch64_AM::LSL && ShiftVal <= 5; |
| } |
| |
| case AArch64::ADDWrx: |
| case AArch64::ADDXrx: |
| case AArch64::ADDXrx64: |
| case AArch64::ADDSWrx: |
| case AArch64::ADDSXrx: |
| case AArch64::ADDSXrx64: { |
| unsigned Imm = MI.getOperand(3).getImm(); |
| switch (AArch64_AM::getArithExtendType(Imm)) { |
| default: |
| return false; |
| case AArch64_AM::UXTB: |
| case AArch64_AM::UXTH: |
| case AArch64_AM::UXTW: |
| case AArch64_AM::UXTX: |
| return AArch64_AM::getArithShiftValue(Imm) <= 4; |
| } |
| } |
| |
| case AArch64::SUBWrs: |
| case AArch64::SUBSWrs: { |
| unsigned Imm = MI.getOperand(3).getImm(); |
| unsigned ShiftVal = AArch64_AM::getShiftValue(Imm); |
| return ShiftVal == 0 || |
| (AArch64_AM::getShiftType(Imm) == AArch64_AM::ASR && ShiftVal == 31); |
| } |
| |
| case AArch64::SUBXrs: |
| case AArch64::SUBSXrs: { |
| unsigned Imm = MI.getOperand(3).getImm(); |
| unsigned ShiftVal = AArch64_AM::getShiftValue(Imm); |
| return ShiftVal == 0 || |
| (AArch64_AM::getShiftType(Imm) == AArch64_AM::ASR && ShiftVal == 63); |
| } |
| |
| case AArch64::SUBWrx: |
| case AArch64::SUBXrx: |
| case AArch64::SUBXrx64: |
| case AArch64::SUBSWrx: |
| case AArch64::SUBSXrx: |
| case AArch64::SUBSXrx64: { |
| unsigned Imm = MI.getOperand(3).getImm(); |
| switch (AArch64_AM::getArithExtendType(Imm)) { |
| default: |
| return false; |
| case AArch64_AM::UXTB: |
| case AArch64_AM::UXTH: |
| case AArch64_AM::UXTW: |
| case AArch64_AM::UXTX: |
| return AArch64_AM::getArithShiftValue(Imm) == 0; |
| } |
| } |
| |
| case AArch64::LDRBBroW: |
| case AArch64::LDRBBroX: |
| case AArch64::LDRBroW: |
| case AArch64::LDRBroX: |
| case AArch64::LDRDroW: |
| case AArch64::LDRDroX: |
| case AArch64::LDRHHroW: |
| case AArch64::LDRHHroX: |
| case AArch64::LDRHroW: |
| case AArch64::LDRHroX: |
| case AArch64::LDRQroW: |
| case AArch64::LDRQroX: |
| case AArch64::LDRSBWroW: |
| case AArch64::LDRSBWroX: |
| case AArch64::LDRSBXroW: |
| case AArch64::LDRSBXroX: |
| case AArch64::LDRSHWroW: |
| case AArch64::LDRSHWroX: |
| case AArch64::LDRSHXroW: |
| case AArch64::LDRSHXroX: |
| case AArch64::LDRSWroW: |
| case AArch64::LDRSWroX: |
| case AArch64::LDRSroW: |
| case AArch64::LDRSroX: |
| case AArch64::LDRWroW: |
| case AArch64::LDRWroX: |
| case AArch64::LDRXroW: |
| case AArch64::LDRXroX: |
| case AArch64::PRFMroW: |
| case AArch64::PRFMroX: |
| case AArch64::STRBBroW: |
| case AArch64::STRBBroX: |
| case AArch64::STRBroW: |
| case AArch64::STRBroX: |
| case AArch64::STRDroW: |
| case AArch64::STRDroX: |
| case AArch64::STRHHroW: |
| case AArch64::STRHHroX: |
| case AArch64::STRHroW: |
| case AArch64::STRHroX: |
| case AArch64::STRQroW: |
| case AArch64::STRQroX: |
| case AArch64::STRSroW: |
| case AArch64::STRSroX: |
| case AArch64::STRWroW: |
| case AArch64::STRWroX: |
| case AArch64::STRXroW: |
| case AArch64::STRXroX: { |
| unsigned IsSigned = MI.getOperand(3).getImm(); |
| return !IsSigned; |
| } |
| } |
| } |
| |
| bool AArch64InstrInfo::isSEHInstruction(const MachineInstr &MI) { |
| unsigned Opc = MI.getOpcode(); |
| switch (Opc) { |
| default: |
| return false; |
| case AArch64::SEH_StackAlloc: |
| case AArch64::SEH_SaveFPLR: |
| case AArch64::SEH_SaveFPLR_X: |
| case AArch64::SEH_SaveReg: |
| case AArch64::SEH_SaveReg_X: |
| case AArch64::SEH_SaveRegP: |
| case AArch64::SEH_SaveRegP_X: |
| case AArch64::SEH_SaveFReg: |
| case AArch64::SEH_SaveFReg_X: |
| case AArch64::SEH_SaveFRegP: |
| case AArch64::SEH_SaveFRegP_X: |
| case AArch64::SEH_SetFP: |
| case AArch64::SEH_AddFP: |
| case AArch64::SEH_Nop: |
| case AArch64::SEH_PrologEnd: |
| case AArch64::SEH_EpilogStart: |
| case AArch64::SEH_EpilogEnd: |
| return true; |
| } |
| } |
| |
| bool AArch64InstrInfo::isCoalescableExtInstr(const MachineInstr &MI, |
| unsigned &SrcReg, unsigned &DstReg, |
| unsigned &SubIdx) const { |
| switch (MI.getOpcode()) { |
| default: |
| return false; |
| case AArch64::SBFMXri: // aka sxtw |
| case AArch64::UBFMXri: // aka uxtw |
| // Check for the 32 -> 64 bit extension case, these instructions can do |
| // much more. |
| if (MI.getOperand(2).getImm() != 0 || MI.getOperand(3).getImm() != 31) |
| return false; |
| // This is a signed or unsigned 32 -> 64 bit extension. |
| SrcReg = MI.getOperand(1).getReg(); |
| DstReg = MI.getOperand(0).getReg(); |
| SubIdx = AArch64::sub_32; |
| return true; |
| } |
| } |
| |
| bool AArch64InstrInfo::areMemAccessesTriviallyDisjoint( |
| const MachineInstr &MIa, const MachineInstr &MIb) const { |
| const TargetRegisterInfo *TRI = &getRegisterInfo(); |
| const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr; |
| int64_t OffsetA = 0, OffsetB = 0; |
| unsigned WidthA = 0, WidthB = 0; |
| |
| assert(MIa.mayLoadOrStore() && "MIa must be a load or store."); |
| assert(MIb.mayLoadOrStore() && "MIb must be a load or store."); |
| |
| if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() || |
| MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef()) |
| return false; |
| |
| // Retrieve the base, offset from the base and width. Width |
| // is the size of memory that is being loaded/stored (e.g. 1, 2, 4, 8). If |
| // base are identical, and the offset of a lower memory access + |
| // the width doesn't overlap the offset of a higher memory access, |
| // then the memory accesses are different. |
| if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) && |
| getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) { |
| if (BaseOpA->isIdenticalTo(*BaseOpB)) { |
| int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB; |
| int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA; |
| int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB; |
| if (LowOffset + LowWidth <= HighOffset) |
| return true; |
| } |
| } |
| return false; |
| } |
| |
| bool AArch64InstrInfo::isSchedulingBoundary(const MachineInstr &MI, |
| const MachineBasicBlock *MBB, |
| const MachineFunction &MF) const { |
| if (TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF)) |
| return true; |
| switch (MI.getOpcode()) { |
| case AArch64::HINT: |
| // CSDB hints are scheduling barriers. |
| if (MI.getOperand(0).getImm() == 0x14) |
| return true; |
| break; |
| case AArch64::DSB: |
| case AArch64::ISB: |
| // DSB and ISB also are scheduling barriers. |
| return true; |
| default:; |
| } |
| return isSEHInstruction(MI); |
| } |
| |
| /// analyzeCompare - For a comparison instruction, return the source registers |
| /// in SrcReg and SrcReg2, and the value it compares against in CmpValue. |
| /// Return true if the comparison instruction can be analyzed. |
| bool AArch64InstrInfo::analyzeCompare(const MachineInstr &MI, unsigned &SrcReg, |
| unsigned &SrcReg2, int &CmpMask, |
| int &CmpValue) const { |
| // The first operand can be a frame index where we'd normally expect a |
| // register. |
| assert(MI.getNumOperands() >= 2 && "All AArch64 cmps should have 2 operands"); |
| if (!MI.getOperand(1).isReg()) |
| return false; |
| |
| switch (MI.getOpcode()) { |
| default: |
| break; |
| case AArch64::SUBSWrr: |
| case AArch64::SUBSWrs: |
| case AArch64::SUBSWrx: |
| case AArch64::SUBSXrr: |
| case AArch64::SUBSXrs: |
| case AArch64::SUBSXrx: |
| case AArch64::ADDSWrr: |
| case AArch64::ADDSWrs: |
| case AArch64::ADDSWrx: |
| case AArch64::ADDSXrr: |
| case AArch64::ADDSXrs: |
| case AArch64::ADDSXrx: |
| // Replace SUBSWrr with SUBWrr if NZCV is not used. |
| SrcReg = MI.getOperand(1).getReg(); |
| SrcReg2 = MI.getOperand(2).getReg(); |
| CmpMask = ~0; |
| CmpValue = 0; |
| return true; |
| case AArch64::SUBSWri: |
| case AArch64::ADDSWri: |
| case AArch64::SUBSXri: |
| case AArch64::ADDSXri: |
| SrcReg = MI.getOperand(1).getReg(); |
| SrcReg2 = 0; |
| CmpMask = ~0; |
| // FIXME: In order to convert CmpValue to 0 or 1 |
| CmpValue = MI.getOperand(2).getImm() != 0; |
| return true; |
| case AArch64::ANDSWri: |
| case AArch64::ANDSXri: |
| // ANDS does not use the same encoding scheme as the others xxxS |
| // instructions. |
| SrcReg = MI.getOperand(1).getReg(); |
| SrcReg2 = 0; |
| CmpMask = ~0; |
| // FIXME:The return val type of decodeLogicalImmediate is uint64_t, |
| // while the type of CmpValue is int. When converting uint64_t to int, |
| // the high 32 bits of uint64_t will be lost. |
| // In fact it causes a bug in spec2006-483.xalancbmk |
| // CmpValue is only used to compare with zero in OptimizeCompareInstr |
| CmpValue = AArch64_AM::decodeLogicalImmediate( |
| MI.getOperand(2).getImm(), |
| MI.getOpcode() == AArch64::ANDSWri ? 32 : 64) != 0; |
| return true; |
| } |
| |
| return false; |
| } |
| |
| static bool UpdateOperandRegClass(MachineInstr &Instr) { |
| MachineBasicBlock *MBB = Instr.getParent(); |
| assert(MBB && "Can't get MachineBasicBlock here"); |
| MachineFunction *MF = MBB->getParent(); |
| assert(MF && "Can't get MachineFunction here"); |
| const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo(); |
| const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo(); |
| MachineRegisterInfo *MRI = &MF->getRegInfo(); |
| |
| for (unsigned OpIdx = 0, EndIdx = Instr.getNumOperands(); OpIdx < EndIdx; |
| ++OpIdx) { |
| MachineOperand &MO = Instr.getOperand(OpIdx); |
| const TargetRegisterClass *OpRegCstraints = |
| Instr.getRegClassConstraint(OpIdx, TII, TRI); |
| |
| // If there's no constraint, there's nothing to do. |
| if (!OpRegCstraints) |
| continue; |
| // If the operand is a frame index, there's nothing to do here. |
| // A frame index operand will resolve correctly during PEI. |
| if (MO.isFI()) |
| continue; |
| |
| assert(MO.isReg() && |
| "Operand has register constraints without being a register!"); |
| |
| Register Reg = MO.getReg(); |
| if (Register::isPhysicalRegister(Reg)) { |
| if (!OpRegCstraints->contains(Reg)) |
| return false; |
| } else if (!OpRegCstraints->hasSubClassEq(MRI->getRegClass(Reg)) && |
| !MRI->constrainRegClass(Reg, OpRegCstraints)) |
| return false; |
| } |
| |
| return true; |
| } |
| |
| /// Return the opcode that does not set flags when possible - otherwise |
| /// return the original opcode. The caller is responsible to do the actual |
| /// substitution and legality checking. |
| static unsigned convertToNonFlagSettingOpc(const MachineInstr &MI) { |
| // Don't convert all compare instructions, because for some the zero register |
| // encoding becomes the sp register. |
| bool MIDefinesZeroReg = false; |
| if (MI.definesRegister(AArch64::WZR) || MI.definesRegister(AArch64::XZR)) |
| MIDefinesZeroReg = true; |
| |
| switch (MI.getOpcode()) { |
| default: |
| return MI.getOpcode(); |
| case AArch64::ADDSWrr: |
| return AArch64::ADDWrr; |
| case AArch64::ADDSWri: |
| return MIDefinesZeroReg ? AArch64::ADDSWri : AArch64::ADDWri; |
| case AArch64::ADDSWrs: |
| return MIDefinesZeroReg ? AArch64::ADDSWrs : AArch64::ADDWrs; |
| case AArch64::ADDSWrx: |
| return AArch64::ADDWrx; |
| case AArch64::ADDSXrr: |
| return AArch64::ADDXrr; |
| case AArch64::ADDSXri: |
| return MIDefinesZeroReg ? AArch64::ADDSXri : AArch64::ADDXri; |
| case AArch64::ADDSXrs: |
| return MIDefinesZeroReg ? AArch64::ADDSXrs : AArch64::ADDXrs; |
| case AArch64::ADDSXrx: |
| return AArch64::ADDXrx; |
| case AArch64::SUBSWrr: |
| return AArch64::SUBWrr; |
| case AArch64::SUBSWri: |
| return MIDefinesZeroReg ? AArch64::SUBSWri : AArch64::SUBWri; |
| case AArch64::SUBSWrs: |
| return MIDefinesZeroReg ? AArch64::SUBSWrs : AArch64::SUBWrs; |
| case AArch64::SUBSWrx: |
| return AArch64::SUBWrx; |
| case AArch64::SUBSXrr: |
| return AArch64::SUBXrr; |
| case AArch64::SUBSXri: |
| return MIDefinesZeroReg ? AArch64::SUBSXri : AArch64::SUBXri; |
| case AArch64::SUBSXrs: |
| return MIDefinesZeroReg ? AArch64::SUBSXrs : AArch64::SUBXrs; |
| case AArch64::SUBSXrx: |
| return AArch64::SUBXrx; |
| } |
| } |
| |
| enum AccessKind { AK_Write = 0x01, AK_Read = 0x10, AK_All = 0x11 }; |
| |
| /// True when condition flags are accessed (either by writing or reading) |
| /// on the instruction trace starting at From and ending at To. |
| /// |
| /// Note: If From and To are from different blocks it's assumed CC are accessed |
| /// on the path. |
| static bool areCFlagsAccessedBetweenInstrs( |
| MachineBasicBlock::iterator From, MachineBasicBlock::iterator To, |
| const TargetRegisterInfo *TRI, const AccessKind AccessToCheck = AK_All) { |
| // Early exit if To is at the beginning of the BB. |
| if (To == To->getParent()->begin()) |
| return true; |
| |
| // Check whether the instructions are in the same basic block |
| // If not, assume the condition flags might get modified somewhere. |
| if (To->getParent() != From->getParent()) |
| return true; |
| |
| // From must be above To. |
| assert(std::find_if(++To.getReverse(), To->getParent()->rend(), |
| [From](MachineInstr &MI) { |
| return MI.getIterator() == From; |
| }) != To->getParent()->rend()); |
| |
| // We iterate backward starting \p To until we hit \p From. |
| for (--To; To != From; --To) { |
| const MachineInstr &Instr = *To; |
| |
| if (((AccessToCheck & AK_Write) && |
| Instr.modifiesRegister(AArch64::NZCV, TRI)) || |
| ((AccessToCheck & AK_Read) && Instr.readsRegister(AArch64::NZCV, TRI))) |
| return true; |
| } |
| return false; |
| } |
| |
| /// Try to optimize a compare instruction. A compare instruction is an |
| /// instruction which produces AArch64::NZCV. It can be truly compare |
| /// instruction |
| /// when there are no uses of its destination register. |
| /// |
| /// The following steps are tried in order: |
| /// 1. Convert CmpInstr into an unconditional version. |
| /// 2. Remove CmpInstr if above there is an instruction producing a needed |
| /// condition code or an instruction which can be converted into such an |
| /// instruction. |
| /// Only comparison with zero is supported. |
| bool AArch64InstrInfo::optimizeCompareInstr( |
| MachineInstr &CmpInstr, unsigned SrcReg, unsigned SrcReg2, int CmpMask, |
| int CmpValue, const MachineRegisterInfo *MRI) const { |
| assert(CmpInstr.getParent()); |
| assert(MRI); |
| |
| // Replace SUBSWrr with SUBWrr if NZCV is not used. |
| int DeadNZCVIdx = CmpInstr.findRegisterDefOperandIdx(AArch64::NZCV, true); |
| if (DeadNZCVIdx != -1) { |
| if (CmpInstr.definesRegister(AArch64::WZR) || |
| CmpInstr.definesRegister(AArch64::XZR)) { |
| CmpInstr.eraseFromParent(); |
| return true; |
| } |
| unsigned Opc = CmpInstr.getOpcode(); |
| unsigned NewOpc = convertToNonFlagSettingOpc(CmpInstr); |
| if (NewOpc == Opc) |
| return false; |
| const MCInstrDesc &MCID = get(NewOpc); |
| CmpInstr.setDesc(MCID); |
| CmpInstr.RemoveOperand(DeadNZCVIdx); |
| bool succeeded = UpdateOperandRegClass(CmpInstr); |
| (void)succeeded; |
| assert(succeeded && "Some operands reg class are incompatible!"); |
| return true; |
| } |
| |
| // Continue only if we have a "ri" where immediate is zero. |
| // FIXME:CmpValue has already been converted to 0 or 1 in analyzeCompare |
| // function. |
| assert((CmpValue == 0 || CmpValue == 1) && "CmpValue must be 0 or 1!"); |
| if (CmpValue != 0 || SrcReg2 != 0) |
| return false; |
| |
| // CmpInstr is a Compare instruction if destination register is not used. |
| if (!MRI->use_nodbg_empty(CmpInstr.getOperand(0).getReg())) |
| return false; |
| |
| return substituteCmpToZero(CmpInstr, SrcReg, MRI); |
| } |
| |
| /// Get opcode of S version of Instr. |
| /// If Instr is S version its opcode is returned. |
| /// AArch64::INSTRUCTION_LIST_END is returned if Instr does not have S version |
| /// or we are not interested in it. |
| static unsigned sForm(MachineInstr &Instr) { |
| switch (Instr.getOpcode()) { |
| default: |
| return AArch64::INSTRUCTION_LIST_END; |
| |
| case AArch64::ADDSWrr: |
| case AArch64::ADDSWri: |
| case AArch64::ADDSXrr: |
| case AArch64::ADDSXri: |
| case AArch64::SUBSWrr: |
| case AArch64::SUBSWri: |
| case AArch64::SUBSXrr: |
| case AArch64::SUBSXri: |
| return Instr.getOpcode(); |
| |
| case AArch64::ADDWrr: |
| return AArch64::ADDSWrr; |
| case AArch64::ADDWri: |
| return AArch64::ADDSWri; |
| case AArch64::ADDXrr: |
| return AArch64::ADDSXrr; |
| case AArch64::ADDXri: |
| return AArch64::ADDSXri; |
| case AArch64::ADCWr: |
| return AArch64::ADCSWr; |
| case AArch64::ADCXr: |
| return AArch64::ADCSXr; |
| case AArch64::SUBWrr: |
| return AArch64::SUBSWrr; |
| case AArch64::SUBWri: |
| return AArch64::SUBSWri; |
| case AArch64::SUBXrr: |
| return AArch64::SUBSXrr; |
| case AArch64::SUBXri: |
| return AArch64::SUBSXri; |
| case AArch64::SBCWr: |
| return AArch64::SBCSWr; |
| case AArch64::SBCXr: |
| return AArch64::SBCSXr; |
| case AArch64::ANDWri: |
| return AArch64::ANDSWri; |
| case AArch64::ANDXri: |
| return AArch64::ANDSXri; |
| } |
| } |
| |
| /// Check if AArch64::NZCV should be alive in successors of MBB. |
| static bool areCFlagsAliveInSuccessors(MachineBasicBlock *MBB) { |
| for (auto *BB : MBB->successors()) |
| if (BB->isLiveIn(AArch64::NZCV)) |
| return true; |
| return false; |
| } |
| |
| namespace { |
| |
| struct UsedNZCV { |
| bool N = false; |
| bool Z = false; |
| bool C = false; |
| bool V = false; |
| |
| UsedNZCV() = default; |
| |
| UsedNZCV &operator|=(const UsedNZCV &UsedFlags) { |
| this->N |= UsedFlags.N; |
| this->Z |= UsedFlags.Z; |
| this->C |= UsedFlags.C; |
| this->V |= UsedFlags.V; |
| return *this; |
| } |
| }; |
| |
| } // end anonymous namespace |
| |
| /// Find a condition code used by the instruction. |
| /// Returns AArch64CC::Invalid if either the instruction does not use condition |
| /// codes or we don't optimize CmpInstr in the presence of such instructions. |
| static AArch64CC::CondCode findCondCodeUsedByInstr(const MachineInstr &Instr) { |
| switch (Instr.getOpcode()) { |
| default: |
| return AArch64CC::Invalid; |
| |
| case AArch64::Bcc: { |
| int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV); |
| assert(Idx >= 2); |
| return static_cast<AArch64CC::CondCode>(Instr.getOperand(Idx - 2).getImm()); |
| } |
| |
| case AArch64::CSINVWr: |
| case AArch64::CSINVXr: |
| case AArch64::CSINCWr: |
| case AArch64::CSINCXr: |
| case AArch64::CSELWr: |
| case AArch64::CSELXr: |
| case AArch64::CSNEGWr: |
| case AArch64::CSNEGXr: |
| case AArch64::FCSELSrrr: |
| case AArch64::FCSELDrrr: { |
| int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV); |
| assert(Idx >= 1); |
| return static_cast<AArch64CC::CondCode>(Instr.getOperand(Idx - 1).getImm()); |
| } |
| } |
| } |
| |
| static UsedNZCV getUsedNZCV(AArch64CC::CondCode CC) { |
| assert(CC != AArch64CC::Invalid); |
| UsedNZCV UsedFlags; |
| switch (CC) { |
| default: |
| break; |
| |
| case AArch64CC::EQ: // Z set |
| case AArch64CC::NE: // Z clear |
| UsedFlags.Z = true; |
| break; |
| |
| case AArch64CC::HI: // Z clear and C set |
| case AArch64CC::LS: // Z set or C clear |
| UsedFlags.Z = true; |
| LLVM_FALLTHROUGH; |
| case AArch64CC::HS: // C set |
| case AArch64CC::LO: // C clear |
| UsedFlags.C = true; |
| break; |
| |
| case AArch64CC::MI: // N set |
| case AArch64CC::PL: // N clear |
| UsedFlags.N = true; |
| break; |
| |
| case AArch64CC::VS: // V set |
| case AArch64CC::VC: // V clear |
| UsedFlags.V = true; |
| break; |
| |
| case AArch64CC::GT: // Z clear, N and V the same |
| case AArch64CC::LE: // Z set, N and V differ |
| UsedFlags.Z = true; |
| LLVM_FALLTHROUGH; |
| case AArch64CC::GE: // N and V the same |
| case AArch64CC::LT: // N and V differ |
| UsedFlags.N = true; |
| UsedFlags.V = true; |
| break; |
| } |
| return UsedFlags; |
| } |
| |
| static bool isADDSRegImm(unsigned Opcode) { |
| return Opcode == AArch64::ADDSWri || Opcode == AArch64::ADDSXri; |
| } |
| |
| static bool isSUBSRegImm(unsigned Opcode) { |
| return Opcode == AArch64::SUBSWri || Opcode == AArch64::SUBSXri; |
| } |
| |
| /// Check if CmpInstr can be substituted by MI. |
| /// |
| /// CmpInstr can be substituted: |
| /// - CmpInstr is either 'ADDS %vreg, 0' or 'SUBS %vreg, 0' |
| /// - and, MI and CmpInstr are from the same MachineBB |
| /// - and, condition flags are not alive in successors of the CmpInstr parent |
| /// - and, if MI opcode is the S form there must be no defs of flags between |
| /// MI and CmpInstr |
| /// or if MI opcode is not the S form there must be neither defs of flags |
| /// nor uses of flags between MI and CmpInstr. |
| /// - and C/V flags are not used after CmpInstr |
| static bool canInstrSubstituteCmpInstr(MachineInstr *MI, MachineInstr *CmpInstr, |
| const TargetRegisterInfo *TRI) { |
| assert(MI); |
| assert(sForm(*MI) != AArch64::INSTRUCTION_LIST_END); |
| assert(CmpInstr); |
| |
| const unsigned CmpOpcode = CmpInstr->getOpcode(); |
| if (!isADDSRegImm(CmpOpcode) && !isSUBSRegImm(CmpOpcode)) |
| return false; |
| |
| if (MI->getParent() != CmpInstr->getParent()) |
| return false; |
| |
| if (areCFlagsAliveInSuccessors(CmpInstr->getParent())) |
| return false; |
| |
| AccessKind AccessToCheck = AK_Write; |
| if (sForm(*MI) != MI->getOpcode()) |
| AccessToCheck = AK_All; |
| if (areCFlagsAccessedBetweenInstrs(MI, CmpInstr, TRI, AccessToCheck)) |
| return false; |
| |
| UsedNZCV NZCVUsedAfterCmp; |
| for (auto I = std::next(CmpInstr->getIterator()), |
| E = CmpInstr->getParent()->instr_end(); |
| I != E; ++I) { |
| const MachineInstr &Instr = *I; |
| if (Instr.readsRegister(AArch64::NZCV, TRI)) { |
| AArch64CC::CondCode CC = findCondCodeUsedByInstr(Instr); |
| if (CC == AArch64CC::Invalid) // Unsupported conditional instruction |
| return false; |
| NZCVUsedAfterCmp |= getUsedNZCV(CC); |
| } |
| |
| if (Instr.modifiesRegister(AArch64::NZCV, TRI)) |
| break; |
| } |
| |
| return !NZCVUsedAfterCmp.C && !NZCVUsedAfterCmp.V; |
| } |
| |
| /// Substitute an instruction comparing to zero with another instruction |
| /// which produces needed condition flags. |
| /// |
| /// Return true on success. |
| bool AArch64InstrInfo::substituteCmpToZero( |
| MachineInstr &CmpInstr, unsigned SrcReg, |
| const MachineRegisterInfo *MRI) const { |
| assert(MRI); |
| // Get the unique definition of SrcReg. |
| MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg); |
| if (!MI) |
| return false; |
| |
| const TargetRegisterInfo *TRI = &getRegisterInfo(); |
| |
| unsigned NewOpc = sForm(*MI); |
| if (NewOpc == AArch64::INSTRUCTION_LIST_END) |
| return false; |
| |
| if (!canInstrSubstituteCmpInstr(MI, &CmpInstr, TRI)) |
| return false; |
| |
| // Update the instruction to set NZCV. |
| MI->setDesc(get(NewOpc)); |
| CmpInstr.eraseFromParent(); |
| bool succeeded = UpdateOperandRegClass(*MI); |
| (void)succeeded; |
| assert(succeeded && "Some operands reg class are incompatible!"); |
| MI->addRegisterDefined(AArch64::NZCV, TRI); |
| return true; |
| } |
| |
| bool AArch64InstrInfo::expandPostRAPseudo(MachineInstr &MI) const { |
| if (MI.getOpcode() != TargetOpcode::LOAD_STACK_GUARD && |
| MI.getOpcode() != AArch64::CATCHRET) |
| return false; |
| |
| MachineBasicBlock &MBB = *MI.getParent(); |
| auto &Subtarget = MBB.getParent()->getSubtarget<AArch64Subtarget>(); |
| auto TRI = Subtarget.getRegisterInfo(); |
| DebugLoc DL = MI.getDebugLoc(); |
| |
| if (MI.getOpcode() == AArch64::CATCHRET) { |
| // Skip to the first instruction before the epilog. |
| const TargetInstrInfo *TII = |
| MBB.getParent()->getSubtarget().getInstrInfo(); |
| MachineBasicBlock *TargetMBB = MI.getOperand(0).getMBB(); |
| auto MBBI = MachineBasicBlock::iterator(MI); |
| MachineBasicBlock::iterator FirstEpilogSEH = std::prev(MBBI); |
| while (FirstEpilogSEH->getFlag(MachineInstr::FrameDestroy) && |
| FirstEpilogSEH != MBB.begin()) |
| FirstEpilogSEH = std::prev(FirstEpilogSEH); |
| if (FirstEpilogSEH != MBB.begin()) |
| FirstEpilogSEH = std::next(FirstEpilogSEH); |
| BuildMI(MBB, FirstEpilogSEH, DL, TII->get(AArch64::ADRP)) |
| .addReg(AArch64::X0, RegState::Define) |
| .addMBB(TargetMBB); |
| BuildMI(MBB, FirstEpilogSEH, DL, TII->get(AArch64::ADDXri)) |
| .addReg(AArch64::X0, RegState::Define) |
| .addReg(AArch64::X0) |
| .addMBB(TargetMBB) |
| .addImm(0); |
| return true; |
| } |
| |
| Register Reg = MI.getOperand(0).getReg(); |
| const GlobalValue *GV = |
| cast<GlobalValue>((*MI.memoperands_begin())->getValue()); |
| const TargetMachine &TM = MBB.getParent()->getTarget(); |
| unsigned OpFlags = Subtarget.ClassifyGlobalReference(GV, TM); |
| const unsigned char MO_NC = AArch64II::MO_NC; |
| |
| if ((OpFlags & AArch64II::MO_GOT) != 0) { |
| BuildMI(MBB, MI, DL, get(AArch64::LOADgot), Reg) |
| .addGlobalAddress(GV, 0, OpFlags); |
| if (Subtarget.isTargetILP32()) { |
| unsigned Reg32 = TRI->getSubReg(Reg, AArch64::sub_32); |
| BuildMI(MBB, MI, DL, get(AArch64::LDRWui)) |
| .addDef(Reg32, RegState::Dead) |
| .addUse(Reg, RegState::Kill) |
| .addImm(0) |
| .addMemOperand(*MI.memoperands_begin()) |
| .addDef(Reg, RegState::Implicit); |
| } else { |
| BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg) |
| .addReg(Reg, RegState::Kill) |
| .addImm(0) |
| .addMemOperand(*MI.memoperands_begin()); |
| } |
| } else if (TM.getCodeModel() == CodeModel::Large) { |
| assert(!Subtarget.isTargetILP32() && "how can large exist in ILP32?"); |
| BuildMI(MBB, MI, DL, get(AArch64::MOVZXi), Reg) |
| .addGlobalAddress(GV, 0, AArch64II::MO_G0 | MO_NC) |
| .addImm(0); |
| BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg) |
| .addReg(Reg, RegState::Kill) |
| .addGlobalAddress(GV, 0, AArch64II::MO_G1 | MO_NC) |
| .addImm(16); |
| BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg) |
| .addReg(Reg, RegState::Kill) |
| .addGlobalAddress(GV, 0, AArch64II::MO_G2 | MO_NC) |
| .addImm(32); |
| BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg) |
| .addReg(Reg, RegState::Kill) |
| .addGlobalAddress(GV, 0, AArch64II::MO_G3) |
| .addImm(48); |
| BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg) |
| .addReg(Reg, RegState::Kill) |
| .addImm(0) |
| .addMemOperand(*MI.memoperands_begin()); |
| } else if (TM.getCodeModel() == CodeModel::Tiny) { |
| BuildMI(MBB, MI, DL, get(AArch64::ADR), Reg) |
| .addGlobalAddress(GV, 0, OpFlags); |
| } else { |
| BuildMI(MBB, MI, DL, get(AArch64::ADRP), Reg) |
| .addGlobalAddress(GV, 0, OpFlags | AArch64II::MO_PAGE); |
| unsigned char LoFlags = OpFlags | AArch64II::MO_PAGEOFF | MO_NC; |
| if (Subtarget.isTargetILP32()) { |
| unsigned Reg32 = TRI->getSubReg(Reg, AArch64::sub_32); |
| BuildMI(MBB, MI, DL, get(AArch64::LDRWui)) |
| .addDef(Reg32, RegState::Dead) |
| .addUse(Reg, RegState::Kill) |
| .addGlobalAddress(GV, 0, LoFlags) |
| .addMemOperand(*MI.memoperands_begin()) |
| .addDef(Reg, RegState::Implicit); |
| } else { |
| BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg) |
| .addReg(Reg, RegState::Kill) |
| .addGlobalAddress(GV, 0, LoFlags) |
| .addMemOperand(*MI.memoperands_begin()); |
| } |
| } |
| |
| MBB.erase(MI); |
| |
| return true; |
| } |
| |
| // Return true if this instruction simply sets its single destination register |
| // to zero. This is equivalent to a register rename of the zero-register. |
| bool AArch64InstrInfo::isGPRZero(const MachineInstr &MI) { |
| switch (MI.getOpcode()) { |
| default: |
| break; |
| case AArch64::MOVZWi: |
| case AArch64::MOVZXi: // movz Rd, #0 (LSL #0) |
| if (MI.getOperand(1).isImm() && MI.getOperand(1).getImm() == 0) { |
| assert(MI.getDesc().getNumOperands() == 3 && |
| MI.getOperand(2).getImm() == 0 && "invalid MOVZi operands"); |
| return true; |
| } |
| break; |
| case AArch64::ANDWri: // and Rd, Rzr, #imm |
| return MI.getOperand(1).getReg() == AArch64::WZR; |
| case AArch64::ANDXri: |
| return MI.getOperand(1).getReg() == AArch64::XZR; |
| case TargetOpcode::COPY: |
| return MI.getOperand(1).getReg() == AArch64::WZR; |
| } |
| return false; |
| } |
| |
| // Return true if this instruction simply renames a general register without |
| // modifying bits. |
| bool AArch64InstrInfo::isGPRCopy(const MachineInstr &MI) { |
| switch (MI.getOpcode()) { |
| default: |
| break; |
| case TargetOpcode::COPY: { |
| // GPR32 copies will by lowered to ORRXrs |
| Register DstReg = MI.getOperand(0).getReg(); |
| return (AArch64::GPR32RegClass.contains(DstReg) || |
| AArch64::GPR64RegClass.contains(DstReg)); |
| } |
| case AArch64::ORRXrs: // orr Xd, Xzr, Xm (LSL #0) |
| if (MI.getOperand(1).getReg() == AArch64::XZR) { |
| assert(MI.getDesc().getNumOperands() == 4 && |
| MI.getOperand(3).getImm() == 0 && "invalid ORRrs operands"); |
| return true; |
| } |
| break; |
| case AArch64::ADDXri: // add Xd, Xn, #0 (LSL #0) |
| if (MI.getOperand(2).getImm() == 0) { |
| assert(MI.getDesc().getNumOperands() == 4 && |
| MI.getOperand(3).getImm() == 0 && "invalid ADDXri operands"); |
| return true; |
| } |
| break; |
| } |
| return false; |
| } |
| |
| // Return true if this instruction simply renames a general register without |
| // modifying bits. |
| bool AArch64InstrInfo::isFPRCopy(const MachineInstr &MI) { |
| switch (MI.getOpcode()) { |
| default: |
| break; |
| case TargetOpcode::COPY: { |
| // FPR64 copies will by lowered to ORR.16b |
| Register DstReg = MI.getOperand(0).getReg(); |
| return (AArch64::FPR64RegClass.contains(DstReg) || |
| AArch64::FPR128RegClass.contains(DstReg)); |
| } |
| case AArch64::ORRv16i8: |
| if (MI.getOperand(1).getReg() == MI.getOperand(2).getReg()) { |
| assert(MI.getDesc().getNumOperands() == 3 && MI.getOperand(0).isReg() && |
| "invalid ORRv16i8 operands"); |
| return true; |
| } |
| break; |
| } |
| return false; |
| } |
| |
| unsigned AArch64InstrInfo::isLoadFromStackSlot(const MachineInstr &MI, |
| int &FrameIndex) const { |
| switch (MI.getOpcode()) { |
| default: |
| break; |
| case AArch64::LDRWui: |
| case AArch64::LDRXui: |
| case AArch64::LDRBui: |
| case AArch64::LDRHui: |
| case AArch64::LDRSui: |
| case AArch64::LDRDui: |
| case AArch64::LDRQui: |
| if (MI.getOperand(0).getSubReg() == 0 && MI.getOperand(1).isFI() && |
| MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0) { |
| FrameIndex = MI.getOperand(1).getIndex(); |
| return MI.getOperand(0).getReg(); |
| } |
| break; |
| } |
| |
| return 0; |
| } |
| |
| unsigned AArch64InstrInfo::isStoreToStackSlot(const MachineInstr &MI, |
| int &FrameIndex) const { |
| switch (MI.getOpcode()) { |
| default: |
| break; |
| case AArch64::STRWui: |
| case AArch64::STRXui: |
| case AArch64::STRBui: |
| case AArch64::STRHui: |
| case AArch64::STRSui: |
| case AArch64::STRDui: |
| case AArch64::STRQui: |
| if (MI.getOperand(0).getSubReg() == 0 && MI.getOperand(1).isFI() && |
| MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0) { |
| FrameIndex = MI.getOperand(1).getIndex(); |
| return MI.getOperand(0).getReg(); |
| } |
| break; |
| } |
| return 0; |
| } |
| |
| /// Check all MachineMemOperands for a hint to suppress pairing. |
| bool AArch64InstrInfo::isLdStPairSuppressed(const MachineInstr &MI) { |
| return llvm::any_of(MI.memoperands(), [](MachineMemOperand *MMO) { |
| return MMO->getFlags() & MOSuppressPair; |
| }); |
| } |
| |
| /// Set a flag on the first MachineMemOperand to suppress pairing. |
| void AArch64InstrInfo::suppressLdStPair(MachineInstr &MI) { |
| if (MI.memoperands_empty()) |
| return; |
| (*MI.memoperands_begin())->setFlags(MOSuppressPair); |
| } |
| |
| /// Check all MachineMemOperands for a hint that the load/store is strided. |
| bool AArch64InstrInfo::isStridedAccess(const MachineInstr &MI) { |
| return llvm::any_of(MI.memoperands(), [](MachineMemOperand *MMO) { |
| return MMO->getFlags() & MOStridedAccess; |
| }); |
| } |
| |
| bool AArch64InstrInfo::isUnscaledLdSt(unsigned Opc) { |
| switch (Opc) { |
| default: |
| return false; |
| case AArch64::STURSi: |
| case AArch64::STURDi: |
| case AArch64::STURQi: |
| case AArch64::STURBBi: |
| case AArch64::STURHHi: |
| case AArch64::STURWi: |
| case AArch64::STURXi: |
| case AArch64::LDURSi: |
| case AArch64::LDURDi: |
| case AArch64::LDURQi: |
| case AArch64::LDURWi: |
| case AArch64::LDURXi: |
| case AArch64::LDURSWi: |
| case AArch64::LDURHHi: |
| case AArch64::LDURBBi: |
| case AArch64::LDURSBWi: |
| case AArch64::LDURSHWi: |
| return true; |
| } |
| } |
| |
| Optional<unsigned> AArch64InstrInfo::getUnscaledLdSt(unsigned Opc) { |
| switch (Opc) { |
| default: return {}; |
| case AArch64::PRFMui: return AArch64::PRFUMi; |
| case AArch64::LDRXui: return AArch64::LDURXi; |
| case AArch64::LDRWui: return AArch64::LDURWi; |
| case AArch64::LDRBui: return AArch64::LDURBi; |
| case AArch64::LDRHui: return AArch64::LDURHi; |
| case AArch64::LDRSui: return AArch64::LDURSi; |
| case AArch64::LDRDui: return AArch64::LDURDi; |
| case AArch64::LDRQui: return AArch64::LDURQi; |
| case AArch64::LDRBBui: return AArch64::LDURBBi; |
| case AArch64::LDRHHui: return AArch64::LDURHHi; |
| case AArch64::LDRSBXui: return AArch64::LDURSBXi; |
| case AArch64::LDRSBWui: return AArch64::LDURSBWi; |
| case AArch64::LDRSHXui: return AArch64::LDURSHXi; |
| case AArch64::LDRSHWui: return AArch64::LDURSHWi; |
| case AArch64::LDRSWui: return AArch64::LDURSWi; |
| case AArch64::STRXui: return AArch64::STURXi; |
| case AArch64::STRWui: return AArch64::STURWi; |
| case AArch64::STRBui: return AArch64::STURBi; |
| case AArch64::STRHui: return AArch64::STURHi; |
| case AArch64::STRSui: return AArch64::STURSi; |
| case AArch64::STRDui: return AArch64::STURDi; |
| case AArch64::STRQui: return AArch64::STURQi; |
| case AArch64::STRBBui: return AArch64::STURBBi; |
| case AArch64::STRHHui: return AArch64::STURHHi; |
| } |
| } |
| |
| unsigned AArch64InstrInfo::getLoadStoreImmIdx(unsigned Opc) { |
| switch (Opc) { |
| default: |
| return 2; |
| case AArch64::LDPXi: |
| case AArch64::LDPDi: |
| case AArch64::STPXi: |
| case AArch64::STPDi: |
| case AArch64::LDNPXi: |
| case AArch64::LDNPDi: |
| case AArch64::STNPXi: |
| case AArch64::STNPDi: |
| case AArch64::LDPQi: |
| case AArch64::STPQi: |
| case AArch64::LDNPQi: |
| case AArch64::STNPQi: |
| case AArch64::LDPWi: |
| case AArch64::LDPSi: |
| case AArch64::STPWi: |
| case AArch64::STPSi: |
| case AArch64::LDNPWi: |
| case AArch64::LDNPSi: |
| case AArch64::STNPWi: |
| case AArch64::STNPSi: |
| case AArch64::LDG: |
| case AArch64::STGPi: |
| return 3; |
| case AArch64::ADDG: |
| case AArch64::STGOffset: |
| return 2; |
| } |
| } |
| |
| bool AArch64InstrInfo::isPairableLdStInst(const MachineInstr &MI) { |
| switch (MI.getOpcode()) { |
| default: |
| return false; |
| // Scaled instructions. |
| case AArch64::STRSui: |
| case AArch64::STRDui: |
| case AArch64::STRQui: |
| case AArch64::STRXui: |
| case AArch64::STRWui: |
| case AArch64::LDRSui: |
| case AArch64::LDRDui: |
| case AArch64::LDRQui: |
| case AArch64::LDRXui: |
| case AArch64::LDRWui: |
| case AArch64::LDRSWui: |
| // Unscaled instructions. |
| case AArch64::STURSi: |
| case AArch64::STURDi: |
| case AArch64::STURQi: |
| case AArch64::STURWi: |
| case AArch64::STURXi: |
| case AArch64::LDURSi: |
| case AArch64::LDURDi: |
| case AArch64::LDURQi: |
| case AArch64::LDURWi: |
| case AArch64::LDURXi: |
| case AArch64::LDURSWi: |
| return true; |
| } |
| } |
| |
| unsigned AArch64InstrInfo::convertToFlagSettingOpc(unsigned Opc, |
| bool &Is64Bit) { |
| switch (Opc) { |
| default: |
| llvm_unreachable("Opcode has no flag setting equivalent!"); |
| // 32-bit cases: |
| case AArch64::ADDWri: |
| Is64Bit = false; |
| return AArch64::ADDSWri; |
| case AArch64::ADDWrr: |
| Is64Bit = false; |
| return AArch64::ADDSWrr; |
| case AArch64::ADDWrs: |
| Is64Bit = false; |
| return AArch64::ADDSWrs; |
| case AArch64::ADDWrx: |
| Is64Bit = false; |
| return AArch64::ADDSWrx; |
| case AArch64::ANDWri: |
| Is64Bit = false; |
| return AArch64::ANDSWri; |
| case AArch64::ANDWrr: |
| Is64Bit = false; |
| return AArch64::ANDSWrr; |
| case AArch64::ANDWrs: |
| Is64Bit = false; |
| return AArch64::ANDSWrs; |
| case AArch64::BICWrr: |
| Is64Bit = false; |
| return AArch64::BICSWrr; |
| case AArch64::BICWrs: |
| Is64Bit = false; |
| return AArch64::BICSWrs; |
| case AArch64::SUBWri: |
| Is64Bit = false; |
| return AArch64::SUBSWri; |
| case AArch64::SUBWrr: |
| Is64Bit = false; |
| return AArch64::SUBSWrr; |
| case AArch64::SUBWrs: |
| Is64Bit = false; |
| return AArch64::SUBSWrs; |
| case AArch64::SUBWrx: |
| Is64Bit = false; |
| return AArch64::SUBSWrx; |
| // 64-bit cases: |
| case AArch64::ADDXri: |
| Is64Bit = true; |
| return AArch64::ADDSXri; |
| case AArch64::ADDXrr: |
| Is64Bit = true; |
| return AArch64::ADDSXrr; |
| case AArch64::ADDXrs: |
| Is64Bit = true; |
| return AArch64::ADDSXrs; |
| case AArch64::ADDXrx: |
| Is64Bit = true; |
| return AArch64::ADDSXrx; |
| case AArch64::ANDXri: |
| Is64Bit = true; |
| return AArch64::ANDSXri; |
| case AArch64::ANDXrr: |
| Is64Bit = true; |
| return AArch64::ANDSXrr; |
| case AArch64::ANDXrs: |
| Is64Bit = true; |
| return AArch64::ANDSXrs; |
| case AArch64::BICXrr: |
| Is64Bit = true; |
| return AArch64::BICSXrr; |
| case AArch64::BICXrs: |
| Is64Bit = true; |
| return AArch64::BICSXrs; |
| case AArch64::SUBXri: |
| Is64Bit = true; |
| return AArch64::SUBSXri; |
| case AArch64::SUBXrr: |
| Is64Bit = true; |
| return AArch64::SUBSXrr; |
| case AArch64::SUBXrs: |
| Is64Bit = true; |
| return AArch64::SUBSXrs; |
| case AArch64::SUBXrx: |
| Is64Bit = true; |
| return AArch64::SUBSXrx; |
| } |
| } |
| |
| // Is this a candidate for ld/st merging or pairing? For example, we don't |
| // touch volatiles or load/stores that have a hint to avoid pair formation. |
| bool AArch64InstrInfo::isCandidateToMergeOrPair(const MachineInstr &MI) const { |
| // If this is a volatile load/store, don't mess with it. |
| if (MI.hasOrderedMemoryRef()) |
| return false; |
| |
| // Make sure this is a reg/fi+imm (as opposed to an address reloc). |
| assert((MI.getOperand(1).isReg() || MI.getOperand(1).isFI()) && |
| "Expected a reg or frame index operand."); |
| if (!MI.getOperand(2).isImm()) |
| return false; |
| |
| // Can't merge/pair if the instruction modifies the base register. |
| // e.g., ldr x0, [x0] |
| // This case will never occur with an FI base. |
| if (MI.getOperand(1).isReg()) { |
| Register BaseReg = MI.getOperand(1).getReg(); |
| const TargetRegisterInfo *TRI = &getRegisterInfo(); |
| if (MI.modifiesRegister(BaseReg, TRI)) |
| return false; |
| } |
| |
| // Check if this load/store has a hint to avoid pair formation. |
| // MachineMemOperands hints are set by the AArch64StorePairSuppress pass. |
| if (isLdStPairSuppressed(MI)) |
| return false; |
| |
| // Do not pair any callee-save store/reload instructions in the |
| // prologue/epilogue if the CFI information encoded the operations as separate |
| // instructions, as that will cause the size of the actual prologue to mismatch |
| // with the prologue size recorded in the Windows CFI. |
| const MCAsmInfo *MAI = MI.getMF()->getTarget().getMCAsmInfo(); |
| bool NeedsWinCFI = MAI->usesWindowsCFI() && |
| MI.getMF()->getFunction().needsUnwindTableEntry(); |
| if (NeedsWinCFI && (MI.getFlag(MachineInstr::FrameSetup) || |
| MI.getFlag(MachineInstr::FrameDestroy))) |
| return false; |
| |
| // On some CPUs quad load/store pairs are slower than two single load/stores. |
| if (Subtarget.isPaired128Slow()) { |
| switch (MI.getOpcode()) { |
| default: |
| break; |
| case AArch64::LDURQi: |
| case AArch64::STURQi: |
| case AArch64::LDRQui: |
| case AArch64::STRQui: |
| return false; |
| } |
| } |
| |
| return true; |
| } |
| |
| bool AArch64InstrInfo::getMemOperandWithOffset(const MachineInstr &LdSt, |
| const MachineOperand *&BaseOp, |
| int64_t &Offset, |
| const TargetRegisterInfo *TRI) const { |
| if (!LdSt.mayLoadOrStore()) |
| return false; |
| |
| unsigned Width; |
| return getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width, TRI); |
| } |
| |
| bool AArch64InstrInfo::getMemOperandWithOffsetWidth( |
| const MachineInstr &LdSt, const MachineOperand *&BaseOp, int64_t &Offset, |
| unsigned &Width, const TargetRegisterInfo *TRI) const { |
| assert(LdSt.mayLoadOrStore() && "Expected a memory operation."); |
| // Handle only loads/stores with base register followed by immediate offset. |
| if (LdSt.getNumExplicitOperands() == 3) { |
| // Non-paired instruction (e.g., ldr x1, [x0, #8]). |
| if ((!LdSt.getOperand(1).isReg() && !LdSt.getOperand(1).isFI()) || |
| !LdSt.getOperand(2).isImm()) |
| return false; |
| } else if (LdSt.getNumExplicitOperands() == 4) { |
| // Paired instruction (e.g., ldp x1, x2, [x0, #8]). |
| if (!LdSt.getOperand(1).isReg() || |
| (!LdSt.getOperand(2).isReg() && !LdSt.getOperand(2).isFI()) || |
| !LdSt.getOperand(3).isImm()) |
| return false; |
| } else |
| return false; |
| |
| // Get the scaling factor for the instruction and set the width for the |
| // instruction. |
| unsigned Scale = 0; |
| int64_t Dummy1, Dummy2; |
| |
| // If this returns false, then it's an instruction we don't want to handle. |
| if (!getMemOpInfo(LdSt.getOpcode(), Scale, Width, Dummy1, Dummy2)) |
| return false; |
| |
| // Compute the offset. Offset is calculated as the immediate operand |
| // multiplied by the scaling factor. Unscaled instructions have scaling factor |
| // set to 1. |
| if (LdSt.getNumExplicitOperands() == 3) { |
| BaseOp = &LdSt.getOperand(1); |
| Offset = LdSt.getOperand(2).getImm() * Scale; |
| } else { |
| assert(LdSt.getNumExplicitOperands() == 4 && "invalid number of operands"); |
| BaseOp = &LdSt.getOperand(2); |
| Offset = LdSt.getOperand(3).getImm() * Scale; |
| } |
| |
| if (!BaseOp->isReg() && !BaseOp->isFI()) |
| return false; |
| |
| return true; |
| } |
| |
| MachineOperand & |
| AArch64InstrInfo::getMemOpBaseRegImmOfsOffsetOperand(MachineInstr &LdSt) const { |
| assert(LdSt.mayLoadOrStore() && "Expected a memory operation."); |
| MachineOperand &OfsOp = LdSt.getOperand(LdSt.getNumExplicitOperands() - 1); |
| assert(OfsOp.isImm() && "Offset operand wasn't immediate."); |
| return OfsOp; |
| } |
| |
| bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, unsigned &Scale, |
| unsigned &Width, int64_t &MinOffset, |
| int64_t &MaxOffset) { |
| switch (Opcode) { |
| // Not a memory operation or something we want to handle. |
| default: |
| Scale = Width = 0; |
| MinOffset = MaxOffset = 0; |
| return false; |
| case AArch64::STRWpost: |
| case AArch64::LDRWpost: |
| Width = 32; |
| Scale = 4; |
| MinOffset = -256; |
| MaxOffset = 255; |
| break; |
| case AArch64::LDURQi: |
| case AArch64::STURQi: |
| Width = 16; |
| Scale = 1; |
| MinOffset = -256; |
| MaxOffset = 255; |
| break; |
| case AArch64::PRFUMi: |
| case AArch64::LDURXi: |
| case AArch64::LDURDi: |
| case AArch64::STURXi: |
| case AArch64::STURDi: |
| Width = 8; |
| Scale = 1; |
| MinOffset = -256; |
| MaxOffset = 255; |
| break; |
| case AArch64::LDURWi: |
| case AArch64::LDURSi: |
| case AArch64::LDURSWi: |
| case AArch64::STURWi: |
| case AArch64::STURSi: |
| Width = 4; |
| Scale = 1; |
| MinOffset = -256; |
| MaxOffset = 255; |
| break; |
| case AArch64::LDURHi: |
| case AArch64::LDURHHi: |
| case AArch64::LDURSHXi: |
| case AArch64::LDURSHWi: |
| case AArch64::STURHi: |
| case AArch64::STURHHi: |
| Width = 2; |
| Scale = 1; |
| MinOffset = -256; |
| MaxOffset = 255; |
| break; |
| case AArch64::LDURBi: |
| case AArch64::LDURBBi: |
| case AArch64::LDURSBXi: |
| case AArch64::LDURSBWi: |
| case AArch64::STURBi: |
| case AArch64::STURBBi: |
| Width = 1; |
| Scale = 1; |
| MinOffset = -256; |
| MaxOffset = 255; |
| break; |
| case AArch64::LDPQi: |
| case AArch64::LDNPQi: |
| case AArch64::STPQi: |
| case AArch64::STNPQi: |
| Scale = 16; |
| Width = 32; |
| MinOffset = -64; |
| MaxOffset = 63; |
| break; |
| case AArch64::LDRQui: |
| case AArch64::STRQui: |
| Scale = Width = 16; |
| MinOffset = 0; |
| MaxOffset = 4095; |
| break; |
| case AArch64::LDPXi: |
| case AArch64::LDPDi: |
| case AArch64::LDNPXi: |
| case AArch64::LDNPDi: |
| case AArch64::STPXi: |
| case AArch64::STPDi: |
| case AArch64::STNPXi: |
| case AArch64::STNPDi: |
| Scale = 8; |
| Width = 16; |
| MinOffset = -64; |
| MaxOffset = 63; |
| break; |
| case AArch64::PRFMui: |
| case AArch64::LDRXui: |
| case AArch64::LDRDui: |
| case AArch64::STRXui: |
| case AArch64::STRDui: |
| Scale = Width = 8; |
| MinOffset = 0; |
| MaxOffset = 4095; |
| break; |
| case AArch64::LDPWi: |
| case AArch64::LDPSi: |
| case AArch64::LDNPWi: |
| case AArch64::LDNPSi: |
| case AArch64::STPWi: |
| case AArch64::STPSi: |
| case AArch64::STNPWi: |
| case AArch64::STNPSi: |
| Scale = 4; |
| Width = 8; |
| MinOffset = -64; |
| MaxOffset = 63; |
| break; |
| case AArch64::LDRWui: |
| case AArch64::LDRSui: |
| case AArch64::LDRSWui: |
| case AArch64::STRWui: |
| case AArch64::STRSui: |
| Scale = Width = 4; |
| MinOffset = 0; |
| MaxOffset = 4095; |
| break; |
| case AArch64::LDRHui: |
| case AArch64::LDRHHui: |
| case AArch64::LDRSHWui: |
| case AArch64::LDRSHXui: |
| case AArch64::STRHui: |
| case AArch64::STRHHui: |
| Scale = Width = 2; |
| MinOffset = 0; |
| MaxOffset = 4095; |
| break; |
| case AArch64::LDRBui: |
| case AArch64::LDRBBui: |
| case AArch64::LDRSBWui: |
| case AArch64::LDRSBXui: |
| case AArch64::STRBui: |
| case AArch64::STRBBui: |
| Scale = Width = 1; |
| MinOffset = 0; |
| MaxOffset = 4095; |
| break; |
| case AArch64::ADDG: |
| Scale = 16; |
| Width = 0; |
| MinOffset = 0; |
| MaxOffset = 63; |
| break; |
| case AArch64::TAGPstack: |
| Scale = 16; |
| Width = 0; |
| // TAGP with a negative offset turns into SUBP, which has a maximum offset |
| // of 63 (not 64!). |
| MinOffset = -63; |
| MaxOffset = 63; |
| break; |
| case AArch64::LDG: |
| case AArch64::STGOffset: |
| case AArch64::STZGOffset: |
| Scale = Width = 16; |
| MinOffset = -256; |
| MaxOffset = 255; |
| break; |
| case AArch64::LDR_PXI: |
| case AArch64::STR_PXI: |
| Scale = Width = 2; |
| MinOffset = -256; |
| MaxOffset = 255; |
| break; |
| case AArch64::LDR_ZXI: |
| case AArch64::STR_ZXI: |
| Scale = Width = 16; |
| MinOffset = -256; |
| MaxOffset = 255; |
| break; |
| case AArch64::ST2GOffset: |
| case AArch64::STZ2GOffset: |
| Scale = 16; |
| Width = 32; |
| MinOffset = -256; |
| MaxOffset = 255; |
| break; |
| case AArch64::STGPi: |
| Scale = Width = 16; |
| MinOffset = -64; |
| MaxOffset = 63; |
| break; |
| } |
| |
| return true; |
| } |
| |
| // Scaling factor for unscaled load or store. |
| int AArch64InstrInfo::getMemScale(unsigned Opc) { |
| switch (Opc) { |
| default: |
| llvm_unreachable("Opcode has unknown scale!"); |
| case AArch64::LDRBBui: |
| case AArch64::LDURBBi: |
| case AArch64::LDRSBWui: |
| case AArch64::LDURSBWi: |
| case AArch64::STRBBui: |
| case AArch64::STURBBi: |
| return 1; |
| case AArch64::LDRHHui: |
| case AArch64::LDURHHi: |
| case AArch64::LDRSHWui: |
| case AArch64::LDURSHWi: |
| case AArch64::STRHHui: |
| case AArch64::STURHHi: |
| return 2; |
| case AArch64::LDRSui: |
| case AArch64::LDURSi: |
| case AArch64::LDRSWui: |
| case AArch64::LDURSWi: |
| case AArch64::LDRWui: |
| case AArch64::LDURWi: |
| case AArch64::STRSui: |
| case AArch64::STURSi: |
| case AArch64::STRWui: |
| case AArch64::STURWi: |
| case AArch64::LDPSi: |
| case AArch64::LDPSWi: |
| case AArch64::LDPWi: |
| case AArch64::STPSi: |
| case AArch64::STPWi: |
| return 4; |
| case AArch64::LDRDui: |
| case AArch64::LDURDi: |
| case AArch64::LDRXui: |
| case AArch64::LDURXi: |
| case AArch64::STRDui: |
| case AArch64::STURDi: |
| case AArch64::STRXui: |
| case AArch64::STURXi: |
| case AArch64::LDPDi: |
| case AArch64::LDPXi: |
| case AArch64::STPDi: |
| case AArch64::STPXi: |
| return 8; |
| case AArch64::LDRQui: |
| case AArch64::LDURQi: |
| case AArch64::STRQui: |
| case AArch64::STURQi: |
| case AArch64::LDPQi: |
| case AArch64::STPQi: |
| case AArch64::STGOffset: |
| case AArch64::STZGOffset: |
| case AArch64::ST2GOffset: |
| case AArch64::STZ2GOffset: |
| case AArch64::STGPi: |
| return 16; |
| } |
| } |
| |
| // Scale the unscaled offsets. Returns false if the unscaled offset can't be |
| // scaled. |
| static bool scaleOffset(unsigned Opc, int64_t &Offset) { |
| int Scale = AArch64InstrInfo::getMemScale(Opc); |
| |
| // If the byte-offset isn't a multiple of the stride, we can't scale this |
| // offset. |
| if (Offset % Scale != 0) |
| return false; |
| |
| // Convert the byte-offset used by unscaled into an "element" offset used |
| // by the scaled pair load/store instructions. |
| Offset /= Scale; |
| return true; |
| } |
| |
| static bool canPairLdStOpc(unsigned FirstOpc, unsigned SecondOpc) { |
| if (FirstOpc == SecondOpc) |
| return true; |
| // We can also pair sign-ext and zero-ext instructions. |
| switch (FirstOpc) { |
| default: |
| return false; |
| case AArch64::LDRWui: |
| case AArch64::LDURWi: |
| return SecondOpc == AArch64::LDRSWui || SecondOpc == AArch64::LDURSWi; |
| case AArch64::LDRSWui: |
| case AArch64::LDURSWi: |
| return SecondOpc == AArch64::LDRWui || SecondOpc == AArch64::LDURWi; |
| } |
| // These instructions can't be paired based on their opcodes. |
| return false; |
| } |
| |
| static bool shouldClusterFI(const MachineFrameInfo &MFI, int FI1, |
| int64_t Offset1, unsigned Opcode1, int FI2, |
| int64_t Offset2, unsigned Opcode2) { |
| // Accesses through fixed stack object frame indices may access a different |
| // fixed stack slot. Check that the object offsets + offsets match. |
| if (MFI.isFixedObjectIndex(FI1) && MFI.isFixedObjectIndex(FI2)) { |
| int64_t ObjectOffset1 = MFI.getObjectOffset(FI1); |
| int64_t ObjectOffset2 = MFI.getObjectOffset(FI2); |
| assert(ObjectOffset1 <= ObjectOffset2 && "Object offsets are not ordered."); |
| // Convert to scaled object offsets. |
| int Scale1 = AArch64InstrInfo::getMemScale(Opcode1); |
| if (ObjectOffset1 % Scale1 != 0) |
| return false; |
| ObjectOffset1 /= Scale1; |
| int Scale2 = AArch64InstrInfo::getMemScale(Opcode2); |
| if (ObjectOffset2 % Scale2 != 0) |
| return false; |
| ObjectOffset2 /= Scale2; |
| ObjectOffset1 += Offset1; |
| ObjectOffset2 += Offset2; |
| return ObjectOffset1 + 1 == ObjectOffset2; |
| } |
| |
| return FI1 == FI2; |
| } |
| |
| /// Detect opportunities for ldp/stp formation. |
| /// |
| /// Only called for LdSt for which getMemOperandWithOffset returns true. |
| bool AArch64InstrInfo::shouldClusterMemOps(const MachineOperand &BaseOp1, |
| const MachineOperand &BaseOp2, |
| unsigned NumLoads) const { |
| const MachineInstr &FirstLdSt = *BaseOp1.getParent(); |
| const MachineInstr &SecondLdSt = *BaseOp2.getParent(); |
| if (BaseOp1.getType() != BaseOp2.getType()) |
| return false; |
| |
| assert((BaseOp1.isReg() || BaseOp1.isFI()) && |
| "Only base registers and frame indices are supported."); |
| |
| // Check for both base regs and base FI. |
| if (BaseOp1.isReg() && BaseOp1.getReg() != BaseOp2.getReg()) |
| return false; |
| |
| // Only cluster up to a single pair. |
| if (NumLoads > 1) |
| return false; |
| |
| if (!isPairableLdStInst(FirstLdSt) || !isPairableLdStInst(SecondLdSt)) |
| return false; |
| |
| // Can we pair these instructions based on their opcodes? |
| unsigned FirstOpc = FirstLdSt.getOpcode(); |
| unsigned SecondOpc = SecondLdSt.getOpcode(); |
| if (!canPairLdStOpc(FirstOpc, SecondOpc)) |
| return false; |
| |
| // Can't merge volatiles or load/stores that have a hint to avoid pair |
| // formation, for example. |
| if (!isCandidateToMergeOrPair(FirstLdSt) || |
| !isCandidateToMergeOrPair(SecondLdSt)) |
| return false; |
| |
| // isCandidateToMergeOrPair guarantees that operand 2 is an immediate. |
| int64_t Offset1 = FirstLdSt.getOperand(2).getImm(); |
| if (isUnscaledLdSt(FirstOpc) && !scaleOffset(FirstOpc, Offset1)) |
| return false; |
| |
| int64_t Offset2 = SecondLdSt.getOperand(2).getImm(); |
| if (isUnscaledLdSt(SecondOpc) && !scaleOffset(SecondOpc, Offset2)) |
| return false; |
| |
| // Pairwise instructions have a 7-bit signed offset field. |
| if (Offset1 > 63 || Offset1 < -64) |
| return false; |
| |
| // The caller should already have ordered First/SecondLdSt by offset. |
| // Note: except for non-equal frame index bases |
| if (BaseOp1.isFI()) { |
| assert((!BaseOp1.isIdenticalTo(BaseOp2) || Offset1 <= Offset2) && |
| "Caller should have ordered offsets."); |
| |
| const MachineFrameInfo &MFI = |
| FirstLdSt.getParent()->getParent()->getFrameInfo(); |
| return shouldClusterFI(MFI, BaseOp1.getIndex(), Offset1, FirstOpc, |
| BaseOp2.getIndex(), Offset2, SecondOpc); |
| } |
| |
| assert(Offset1 <= Offset2 && "Caller should have ordered offsets."); |
| |
| return Offset1 + 1 == Offset2; |
| } |
| |
| static const MachineInstrBuilder &AddSubReg(const MachineInstrBuilder &MIB, |
| unsigned Reg, unsigned SubIdx, |
| unsigned State, |
| const TargetRegisterInfo *TRI) { |
| if (!SubIdx) |
| return MIB.addReg(Reg, State); |
| |
| if (Register::isPhysicalRegister(Reg)) |
| return MIB.addReg(TRI->getSubReg(Reg, SubIdx), State); |
| return MIB.addReg(Reg, State, SubIdx); |
| } |
| |
| static bool forwardCopyWillClobberTuple(unsigned DestReg, unsigned SrcReg, |
| unsigned NumRegs) { |
| // We really want the positive remainder mod 32 here, that happens to be |
| // easily obtainable with a mask. |
| return ((DestReg - SrcReg) & 0x1f) < NumRegs; |
| } |
| |
| void AArch64InstrInfo::copyPhysRegTuple(MachineBasicBlock &MBB, |
| MachineBasicBlock::iterator I, |
| const DebugLoc &DL, MCRegister DestReg, |
| MCRegister SrcReg, bool KillSrc, |
| unsigned Opcode, |
| ArrayRef<unsigned> Indices) const { |
| assert(Subtarget.hasNEON() && "Unexpected register copy without NEON"); |
| const TargetRegisterInfo *TRI = &getRegisterInfo(); |
| uint16_t DestEncoding = TRI->getEncodingValue(DestReg); |
| uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg); |
| unsigned NumRegs = Indices.size(); |
| |
| int SubReg = 0, End = NumRegs, Incr = 1; |
| if (forwardCopyWillClobberTuple(DestEncoding, SrcEncoding, NumRegs)) { |
| SubReg = NumRegs - 1; |
| End = -1; |
| Incr = -1; |
| } |
| |
| for (; SubReg != End; SubReg += Incr) { |
| const MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(Opcode)); |
| AddSubReg(MIB, DestReg, Indices[SubReg], RegState::Define, TRI); |
| AddSubReg(MIB, SrcReg, Indices[SubReg], 0, TRI); |
| AddSubReg(MIB, SrcReg, Indices[SubReg], getKillRegState(KillSrc), TRI); |
| } |
| } |
| |
| void AArch64InstrInfo::copyGPRRegTuple(MachineBasicBlock &MBB, |
| MachineBasicBlock::iterator I, |
| DebugLoc DL, unsigned DestReg, |
| unsigned SrcReg, bool KillSrc, |
| unsigned Opcode, unsigned ZeroReg, |
| llvm::ArrayRef<unsigned> Indices) const { |
| const TargetRegisterInfo *TRI = &getRegisterInfo(); |
| unsigned NumRegs = Indices.size(); |
| |
| #ifndef NDEBUG |
| uint16_t DestEncoding = TRI->getEncodingValue(DestReg); |
| uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg); |
| assert(DestEncoding % NumRegs == 0 && SrcEncoding % NumRegs == 0 && |
| "GPR reg sequences should not be able to overlap"); |
| #endif |
| |
| for (unsigned SubReg = 0; SubReg != NumRegs; ++SubReg) { |
| const MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(Opcode)); |
| AddSubReg(MIB, DestReg, Indices[SubReg], RegState::Define, TRI); |
| MIB.addReg(ZeroReg); |
| AddSubReg(MIB, SrcReg, Indices[SubReg], getKillRegState(KillSrc), TRI); |
| MIB.addImm(0); |
| } |
| } |
| |
| void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB, |
| MachineBasicBlock::iterator I, |
| const DebugLoc &DL, MCRegister DestReg, |
| MCRegister SrcReg, bool KillSrc) const { |
| if (AArch64::GPR32spRegClass.contains(DestReg) && |
| (AArch64::GPR32spRegClass.contains(SrcReg) || SrcReg == AArch64::WZR)) { |
| const TargetRegisterInfo *TRI = &getRegisterInfo(); |
| |
| if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) { |
| // If either operand is WSP, expand to ADD #0. |
| if (Subtarget.hasZeroCycleRegMove()) { |
| // Cyclone recognizes "ADD Xd, Xn, #0" as a zero-cycle register move. |
| MCRegister DestRegX = TRI->getMatchingSuperReg( |
| DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass); |
| MCRegister SrcRegX = TRI->getMatchingSuperReg( |
| SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass); |
| // This instruction is reading and writing X registers. This may upset |
| // the register scavenger and machine verifier, so we need to indicate |
| // that we are reading an undefined value from SrcRegX, but a proper |
| // value from SrcReg. |
| BuildMI(MBB, I, DL, get(AArch64::ADDXri), DestRegX) |
| .addReg(SrcRegX, RegState::Undef) |
| .addImm(0) |
| .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0)) |
| .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc)); |
| } else { |
| BuildMI(MBB, I, DL, get(AArch64::ADDWri), DestReg) |
| .addReg(SrcReg, getKillRegState(KillSrc)) |
| .addImm(0) |
| .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0)); |
| } |
| } else if (SrcReg == AArch64::WZR && Subtarget.hasZeroCycleZeroingGP()) { |
| BuildMI(MBB, I, DL, get(AArch64::MOVZWi), DestReg) |
| .addImm(0) |
| .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0)); |
| } else { |
| if (Subtarget.hasZeroCycleRegMove()) { |
| // Cyclone recognizes "ORR Xd, XZR, Xm" as a zero-cycle register move. |
| MCRegister DestRegX = TRI->getMatchingSuperReg( |
| DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass); |
| MCRegister SrcRegX = TRI->getMatchingSuperReg( |
| SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass); |
| // This instruction is reading and writing X registers. This may upset |
| // the register scavenger and machine verifier, so we need to indicate |
| // that we are reading an undefined value from SrcRegX, but a proper |
| // value from SrcReg. |
| BuildMI(MBB, I, DL, get(AArch64::ORRXrr), DestRegX) |
| .addReg(AArch64::XZR) |
| .addReg(SrcRegX, RegState::Undef) |
| .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc)); |
| } else { |
| // Otherwise, expand to ORR WZR. |
| BuildMI(MBB, I, DL, get(AArch64::ORRWrr), DestReg) |
| .addReg(AArch64::WZR) |
| .addReg(SrcReg, getKillRegState(KillSrc)); |
| } |
| } |
| return; |
| } |
| |
| // Copy a Predicate register by ORRing with itself. |
| if (AArch64::PPRRegClass.contains(DestReg) && |
| AArch64::PPRRegClass.contains(SrcReg)) { |
| assert(Subtarget.hasSVE() && "Unexpected SVE register."); |
| BuildMI(MBB, I, DL, get(AArch64::ORR_PPzPP), DestReg) |
| .addReg(SrcReg) // Pg |
| .addReg(SrcReg) |
| .addReg(SrcReg, getKillRegState(KillSrc)); |
| return; |
| } |
| |
| // Copy a Z register by ORRing with itself. |
| if (AArch64::ZPRRegClass.contains(DestReg) && |
| AArch64::ZPRRegClass.contains(SrcReg)) { |
| assert(Subtarget.hasSVE() && "Unexpected SVE register."); |
| BuildMI(MBB, I, DL, get(AArch64::ORR_ZZZ), DestReg) |
| .addReg(SrcReg) |
| .addReg(SrcReg, getKillRegState(KillSrc)); |
| return; |
| } |
| |
| if (AArch64::GPR64spRegClass.contains(DestReg) && |
| (AArch64::GPR64spRegClass.contains(SrcReg) || SrcReg == AArch64::XZR)) { |
| if (DestReg == AArch64::SP || SrcReg == AArch64::SP) { |
| // If either operand is SP, expand to ADD #0. |
| BuildMI(MBB, I, DL, get(AArch64::ADDXri), DestReg) |
| .addReg(SrcReg, getKillRegState(KillSrc)) |
| .addImm(0) |
| .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0)); |
| } else if (SrcReg == AArch64::XZR && Subtarget.hasZeroCycleZeroingGP()) { |
| BuildMI(MBB, I, DL, get(AArch64::MOVZXi), DestReg) |
| .addImm(0) |
| .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0)); |
| } else { |
| // Otherwise, expand to ORR XZR. |
| BuildMI(MBB, I, DL, get(AArch64::ORRXrr), DestReg) |
| .addReg(AArch64::XZR) |
| .addReg(SrcReg, getKillRegState(KillSrc)); |
| } |
| return; |
| } |
| |
| // Copy a DDDD register quad by copying the individual sub-registers. |
| if (AArch64::DDDDRegClass.contains(DestReg) && |
| AArch64::DDDDRegClass.contains(SrcReg)) { |
| static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1, |
| AArch64::dsub2, AArch64::dsub3}; |
| copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8, |
| Indices); |
| return; |
| } |
| |
| // Copy a DDD register triple by copying the individual sub-registers. |
| if (AArch64::DDDRegClass.contains(DestReg) && |
| AArch64::DDDRegClass.contains(SrcReg)) { |
| static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1, |
| AArch64::dsub2}; |
| copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8, |
| Indices); |
| return; |
| } |
| |
| // Copy a DD register pair by copying the individual sub-registers. |
| if (AArch64::DDRegClass.contains(DestReg) && |
| AArch64::DDRegClass.contains(SrcReg)) { |
| static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1}; |
| copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8, |
| Indices); |
| return; |
| } |
| |
| // Copy a QQQQ register quad by copying the individual sub-registers. |
| if (AArch64::QQQQRegClass.contains(DestReg) && |
| AArch64::QQQQRegClass.contains(SrcReg)) { |
| static const |