| //===- AArch64LoadStoreOptimizer.cpp - AArch64 load/store opt. pass -------===// | |
| // | |
| // The LLVM Compiler Infrastructure | |
| // | |
| // This file is distributed under the University of Illinois Open Source | |
| // License. See LICENSE.TXT for details. | |
| // | |
| //===----------------------------------------------------------------------===// | |
| // | |
| // This file contains a pass that performs load / store related peephole | |
| // optimizations. This pass should be run after register allocation. | |
| // | |
| //===----------------------------------------------------------------------===// | |
| #include "AArch64InstrInfo.h" | |
| #include "AArch64Subtarget.h" | |
| #include "MCTargetDesc/AArch64AddressingModes.h" | |
| #include "llvm/ADT/BitVector.h" | |
| #include "llvm/ADT/SmallVector.h" | |
| #include "llvm/ADT/Statistic.h" | |
| #include "llvm/ADT/StringRef.h" | |
| #include "llvm/ADT/iterator_range.h" | |
| #include "llvm/Analysis/AliasAnalysis.h" | |
| #include "llvm/CodeGen/MachineBasicBlock.h" | |
| #include "llvm/CodeGen/MachineFunction.h" | |
| #include "llvm/CodeGen/MachineFunctionPass.h" | |
| #include "llvm/CodeGen/MachineInstr.h" | |
| #include "llvm/CodeGen/MachineInstrBuilder.h" | |
| #include "llvm/CodeGen/MachineOperand.h" | |
| #include "llvm/CodeGen/TargetRegisterInfo.h" | |
| #include "llvm/IR/DebugLoc.h" | |
| #include "llvm/MC/MCRegisterInfo.h" | |
| #include "llvm/Pass.h" | |
| #include "llvm/Support/CommandLine.h" | |
| #include "llvm/Support/Debug.h" | |
| #include "llvm/Support/ErrorHandling.h" | |
| #include "llvm/Support/raw_ostream.h" | |
| #include <cassert> | |
| #include <cstdint> | |
| #include <iterator> | |
| #include <limits> | |
| using namespace llvm; | |
| #define DEBUG_TYPE "aarch64-ldst-opt" | |
| STATISTIC(NumPairCreated, "Number of load/store pair instructions generated"); | |
| STATISTIC(NumPostFolded, "Number of post-index updates folded"); | |
| STATISTIC(NumPreFolded, "Number of pre-index updates folded"); | |
| STATISTIC(NumUnscaledPairCreated, | |
| "Number of load/store from unscaled generated"); | |
| STATISTIC(NumZeroStoresPromoted, "Number of narrow zero stores promoted"); | |
| STATISTIC(NumLoadsFromStoresPromoted, "Number of loads from stores promoted"); | |
| // The LdStLimit limits how far we search for load/store pairs. | |
| static cl::opt<unsigned> LdStLimit("aarch64-load-store-scan-limit", | |
| cl::init(20), cl::Hidden); | |
| // The UpdateLimit limits how far we search for update instructions when we form | |
| // pre-/post-index instructions. | |
| static cl::opt<unsigned> UpdateLimit("aarch64-update-scan-limit", cl::init(100), | |
| cl::Hidden); | |
| #define AARCH64_LOAD_STORE_OPT_NAME "AArch64 load / store optimization pass" | |
| namespace { | |
| using LdStPairFlags = struct LdStPairFlags { | |
| // If a matching instruction is found, MergeForward is set to true if the | |
| // merge is to remove the first instruction and replace the second with | |
| // a pair-wise insn, and false if the reverse is true. | |
| bool MergeForward = false; | |
| // SExtIdx gives the index of the result of the load pair that must be | |
| // extended. The value of SExtIdx assumes that the paired load produces the | |
| // value in this order: (I, returned iterator), i.e., -1 means no value has | |
| // to be extended, 0 means I, and 1 means the returned iterator. | |
| int SExtIdx = -1; | |
| LdStPairFlags() = default; | |
| void setMergeForward(bool V = true) { MergeForward = V; } | |
| bool getMergeForward() const { return MergeForward; } | |
| void setSExtIdx(int V) { SExtIdx = V; } | |
| int getSExtIdx() const { return SExtIdx; } | |
| }; | |
| struct AArch64LoadStoreOpt : public MachineFunctionPass { | |
| static char ID; | |
| AArch64LoadStoreOpt() : MachineFunctionPass(ID) { | |
| initializeAArch64LoadStoreOptPass(*PassRegistry::getPassRegistry()); | |
| } | |
| AliasAnalysis *AA; | |
| const AArch64InstrInfo *TII; | |
| const TargetRegisterInfo *TRI; | |
| const AArch64Subtarget *Subtarget; | |
| // Track which register units have been modified and used. | |
| LiveRegUnits ModifiedRegUnits, UsedRegUnits; | |
| void getAnalysisUsage(AnalysisUsage &AU) const override { | |
| AU.addRequired<AAResultsWrapperPass>(); | |
| MachineFunctionPass::getAnalysisUsage(AU); | |
| } | |
| // Scan the instructions looking for a load/store that can be combined | |
| // with the current instruction into a load/store pair. | |
| // Return the matching instruction if one is found, else MBB->end(). | |
| MachineBasicBlock::iterator findMatchingInsn(MachineBasicBlock::iterator I, | |
| LdStPairFlags &Flags, | |
| unsigned Limit, | |
| bool FindNarrowMerge); | |
| // Scan the instructions looking for a store that writes to the address from | |
| // which the current load instruction reads. Return true if one is found. | |
| bool findMatchingStore(MachineBasicBlock::iterator I, unsigned Limit, | |
| MachineBasicBlock::iterator &StoreI); | |
| // Merge the two instructions indicated into a wider narrow store instruction. | |
| MachineBasicBlock::iterator | |
| mergeNarrowZeroStores(MachineBasicBlock::iterator I, | |
| MachineBasicBlock::iterator MergeMI, | |
| const LdStPairFlags &Flags); | |
| // Merge the two instructions indicated into a single pair-wise instruction. | |
| MachineBasicBlock::iterator | |
| mergePairedInsns(MachineBasicBlock::iterator I, | |
| MachineBasicBlock::iterator Paired, | |
| const LdStPairFlags &Flags); | |
| // Promote the load that reads directly from the address stored to. | |
| MachineBasicBlock::iterator | |
| promoteLoadFromStore(MachineBasicBlock::iterator LoadI, | |
| MachineBasicBlock::iterator StoreI); | |
| // Scan the instruction list to find a base register update that can | |
| // be combined with the current instruction (a load or store) using | |
| // pre or post indexed addressing with writeback. Scan forwards. | |
| MachineBasicBlock::iterator | |
| findMatchingUpdateInsnForward(MachineBasicBlock::iterator I, | |
| int UnscaledOffset, unsigned Limit); | |
| // Scan the instruction list to find a base register update that can | |
| // be combined with the current instruction (a load or store) using | |
| // pre or post indexed addressing with writeback. Scan backwards. | |
| MachineBasicBlock::iterator | |
| findMatchingUpdateInsnBackward(MachineBasicBlock::iterator I, unsigned Limit); | |
| // Find an instruction that updates the base register of the ld/st | |
| // instruction. | |
| bool isMatchingUpdateInsn(MachineInstr &MemMI, MachineInstr &MI, | |
| unsigned BaseReg, int Offset); | |
| // Merge a pre- or post-index base register update into a ld/st instruction. | |
| MachineBasicBlock::iterator | |
| mergeUpdateInsn(MachineBasicBlock::iterator I, | |
| MachineBasicBlock::iterator Update, bool IsPreIdx); | |
| // Find and merge zero store instructions. | |
| bool tryToMergeZeroStInst(MachineBasicBlock::iterator &MBBI); | |
| // Find and pair ldr/str instructions. | |
| bool tryToPairLdStInst(MachineBasicBlock::iterator &MBBI); | |
| // Find and promote load instructions which read directly from store. | |
| bool tryToPromoteLoadFromStore(MachineBasicBlock::iterator &MBBI); | |
| // Find and merge a base register updates before or after a ld/st instruction. | |
| bool tryToMergeLdStUpdate(MachineBasicBlock::iterator &MBBI); | |
| bool optimizeBlock(MachineBasicBlock &MBB, bool EnableNarrowZeroStOpt); | |
| bool runOnMachineFunction(MachineFunction &Fn) override; | |
| MachineFunctionProperties getRequiredProperties() const override { | |
| return MachineFunctionProperties().set( | |
| MachineFunctionProperties::Property::NoVRegs); | |
| } | |
| StringRef getPassName() const override { return AARCH64_LOAD_STORE_OPT_NAME; } | |
| }; | |
| char AArch64LoadStoreOpt::ID = 0; | |
| } // end anonymous namespace | |
| INITIALIZE_PASS(AArch64LoadStoreOpt, "aarch64-ldst-opt", | |
| AARCH64_LOAD_STORE_OPT_NAME, false, false) | |
| static bool isNarrowStore(unsigned Opc) { | |
| switch (Opc) { | |
| default: | |
| return false; | |
| case AArch64::STRBBui: | |
| case AArch64::STURBBi: | |
| case AArch64::STRHHui: | |
| case AArch64::STURHHi: | |
| return true; | |
| } | |
| } | |
| // Scaling factor for unscaled load or store. | |
| static int getMemScale(MachineInstr &MI) { | |
| switch (MI.getOpcode()) { | |
| default: | |
| llvm_unreachable("Opcode has unknown scale!"); | |
| case AArch64::LDRBBui: | |
| case AArch64::LDURBBi: | |
| case AArch64::LDRSBWui: | |
| case AArch64::LDURSBWi: | |
| case AArch64::STRBBui: | |
| case AArch64::STURBBi: | |
| return 1; | |
| case AArch64::LDRHHui: | |
| case AArch64::LDURHHi: | |
| case AArch64::LDRSHWui: | |
| case AArch64::LDURSHWi: | |
| case AArch64::STRHHui: | |
| case AArch64::STURHHi: | |
| return 2; | |
| case AArch64::LDRSui: | |
| case AArch64::LDURSi: | |
| case AArch64::LDRSWui: | |
| case AArch64::LDURSWi: | |
| case AArch64::LDRWui: | |
| case AArch64::LDURWi: | |
| case AArch64::STRSui: | |
| case AArch64::STURSi: | |
| case AArch64::STRWui: | |
| case AArch64::STURWi: | |
| case AArch64::LDPSi: | |
| case AArch64::LDPSWi: | |
| case AArch64::LDPWi: | |
| case AArch64::STPSi: | |
| case AArch64::STPWi: | |
| return 4; | |
| case AArch64::LDRDui: | |
| case AArch64::LDURDi: | |
| case AArch64::LDRXui: | |
| case AArch64::LDURXi: | |
| case AArch64::STRDui: | |
| case AArch64::STURDi: | |
| case AArch64::STRXui: | |
| case AArch64::STURXi: | |
| case AArch64::LDPDi: | |
| case AArch64::LDPXi: | |
| case AArch64::STPDi: | |
| case AArch64::STPXi: | |
| return 8; | |
| case AArch64::LDRQui: | |
| case AArch64::LDURQi: | |
| case AArch64::STRQui: | |
| case AArch64::STURQi: | |
| case AArch64::LDPQi: | |
| case AArch64::STPQi: | |
| return 16; | |
| } | |
| } | |
| static unsigned getMatchingNonSExtOpcode(unsigned Opc, | |
| bool *IsValidLdStrOpc = nullptr) { | |
| if (IsValidLdStrOpc) | |
| *IsValidLdStrOpc = true; | |
| switch (Opc) { | |
| default: | |
| if (IsValidLdStrOpc) | |
| *IsValidLdStrOpc = false; | |
| return std::numeric_limits<unsigned>::max(); | |
| case AArch64::STRDui: | |
| case AArch64::STURDi: | |
| case AArch64::STRQui: | |
| case AArch64::STURQi: | |
| case AArch64::STRBBui: | |
| case AArch64::STURBBi: | |
| case AArch64::STRHHui: | |
| case AArch64::STURHHi: | |
| case AArch64::STRWui: | |
| case AArch64::STURWi: | |
| case AArch64::STRXui: | |
| case AArch64::STURXi: | |
| case AArch64::LDRDui: | |
| case AArch64::LDURDi: | |
| case AArch64::LDRQui: | |
| case AArch64::LDURQi: | |
| case AArch64::LDRWui: | |
| case AArch64::LDURWi: | |
| case AArch64::LDRXui: | |
| case AArch64::LDURXi: | |
| case AArch64::STRSui: | |
| case AArch64::STURSi: | |
| case AArch64::LDRSui: | |
| case AArch64::LDURSi: | |
| return Opc; | |
| case AArch64::LDRSWui: | |
| return AArch64::LDRWui; | |
| case AArch64::LDURSWi: | |
| return AArch64::LDURWi; | |
| } | |
| } | |
| static unsigned getMatchingWideOpcode(unsigned Opc) { | |
| switch (Opc) { | |
| default: | |
| llvm_unreachable("Opcode has no wide equivalent!"); | |
| case AArch64::STRBBui: | |
| return AArch64::STRHHui; | |
| case AArch64::STRHHui: | |
| return AArch64::STRWui; | |
| case AArch64::STURBBi: | |
| return AArch64::STURHHi; | |
| case AArch64::STURHHi: | |
| return AArch64::STURWi; | |
| case AArch64::STURWi: | |
| return AArch64::STURXi; | |
| case AArch64::STRWui: | |
| return AArch64::STRXui; | |
| } | |
| } | |
| static unsigned getMatchingPairOpcode(unsigned Opc) { | |
| switch (Opc) { | |
| default: | |
| llvm_unreachable("Opcode has no pairwise equivalent!"); | |
| case AArch64::STRSui: | |
| case AArch64::STURSi: | |
| return AArch64::STPSi; | |
| case AArch64::STRDui: | |
| case AArch64::STURDi: | |
| return AArch64::STPDi; | |
| case AArch64::STRQui: | |
| case AArch64::STURQi: | |
| return AArch64::STPQi; | |
| case AArch64::STRWui: | |
| case AArch64::STURWi: | |
| return AArch64::STPWi; | |
| case AArch64::STRXui: | |
| case AArch64::STURXi: | |
| return AArch64::STPXi; | |
| case AArch64::LDRSui: | |
| case AArch64::LDURSi: | |
| return AArch64::LDPSi; | |
| case AArch64::LDRDui: | |
| case AArch64::LDURDi: | |
| return AArch64::LDPDi; | |
| case AArch64::LDRQui: | |
| case AArch64::LDURQi: | |
| return AArch64::LDPQi; | |
| case AArch64::LDRWui: | |
| case AArch64::LDURWi: | |
| return AArch64::LDPWi; | |
| case AArch64::LDRXui: | |
| case AArch64::LDURXi: | |
| return AArch64::LDPXi; | |
| case AArch64::LDRSWui: | |
| case AArch64::LDURSWi: | |
| return AArch64::LDPSWi; | |
| } | |
| } | |
| static unsigned isMatchingStore(MachineInstr &LoadInst, | |
| MachineInstr &StoreInst) { | |
| unsigned LdOpc = LoadInst.getOpcode(); | |
| unsigned StOpc = StoreInst.getOpcode(); | |
| switch (LdOpc) { | |
| default: | |
| llvm_unreachable("Unsupported load instruction!"); | |
| case AArch64::LDRBBui: | |
| return StOpc == AArch64::STRBBui || StOpc == AArch64::STRHHui || | |
| StOpc == AArch64::STRWui || StOpc == AArch64::STRXui; | |
| case AArch64::LDURBBi: | |
| return StOpc == AArch64::STURBBi || StOpc == AArch64::STURHHi || | |
| StOpc == AArch64::STURWi || StOpc == AArch64::STURXi; | |
| case AArch64::LDRHHui: | |
| return StOpc == AArch64::STRHHui || StOpc == AArch64::STRWui || | |
| StOpc == AArch64::STRXui; | |
| case AArch64::LDURHHi: | |
| return StOpc == AArch64::STURHHi || StOpc == AArch64::STURWi || | |
| StOpc == AArch64::STURXi; | |
| case AArch64::LDRWui: | |
| return StOpc == AArch64::STRWui || StOpc == AArch64::STRXui; | |
| case AArch64::LDURWi: | |
| return StOpc == AArch64::STURWi || StOpc == AArch64::STURXi; | |
| case AArch64::LDRXui: | |
| return StOpc == AArch64::STRXui; | |
| case AArch64::LDURXi: | |
| return StOpc == AArch64::STURXi; | |
| } | |
| } | |
| static unsigned getPreIndexedOpcode(unsigned Opc) { | |
| // FIXME: We don't currently support creating pre-indexed loads/stores when | |
| // the load or store is the unscaled version. If we decide to perform such an | |
| // optimization in the future the cases for the unscaled loads/stores will | |
| // need to be added here. | |
| switch (Opc) { | |
| default: | |
| llvm_unreachable("Opcode has no pre-indexed equivalent!"); | |
| case AArch64::STRSui: | |
| return AArch64::STRSpre; | |
| case AArch64::STRDui: | |
| return AArch64::STRDpre; | |
| case AArch64::STRQui: | |
| return AArch64::STRQpre; | |
| case AArch64::STRBBui: | |
| return AArch64::STRBBpre; | |
| case AArch64::STRHHui: | |
| return AArch64::STRHHpre; | |
| case AArch64::STRWui: | |
| return AArch64::STRWpre; | |
| case AArch64::STRXui: | |
| return AArch64::STRXpre; | |
| case AArch64::LDRSui: | |
| return AArch64::LDRSpre; | |
| case AArch64::LDRDui: | |
| return AArch64::LDRDpre; | |
| case AArch64::LDRQui: | |
| return AArch64::LDRQpre; | |
| case AArch64::LDRBBui: | |
| return AArch64::LDRBBpre; | |
| case AArch64::LDRHHui: | |
| return AArch64::LDRHHpre; | |
| case AArch64::LDRWui: | |
| return AArch64::LDRWpre; | |
| case AArch64::LDRXui: | |
| return AArch64::LDRXpre; | |
| case AArch64::LDRSWui: | |
| return AArch64::LDRSWpre; | |
| case AArch64::LDPSi: | |
| return AArch64::LDPSpre; | |
| case AArch64::LDPSWi: | |
| return AArch64::LDPSWpre; | |
| case AArch64::LDPDi: | |
| return AArch64::LDPDpre; | |
| case AArch64::LDPQi: | |
| return AArch64::LDPQpre; | |
| case AArch64::LDPWi: | |
| return AArch64::LDPWpre; | |
| case AArch64::LDPXi: | |
| return AArch64::LDPXpre; | |
| case AArch64::STPSi: | |
| return AArch64::STPSpre; | |
| case AArch64::STPDi: | |
| return AArch64::STPDpre; | |
| case AArch64::STPQi: | |
| return AArch64::STPQpre; | |
| case AArch64::STPWi: | |
| return AArch64::STPWpre; | |
| case AArch64::STPXi: | |
| return AArch64::STPXpre; | |
| } | |
| } | |
| static unsigned getPostIndexedOpcode(unsigned Opc) { | |
| switch (Opc) { | |
| default: | |
| llvm_unreachable("Opcode has no post-indexed wise equivalent!"); | |
| case AArch64::STRSui: | |
| case AArch64::STURSi: | |
| return AArch64::STRSpost; | |
| case AArch64::STRDui: | |
| case AArch64::STURDi: | |
| return AArch64::STRDpost; | |
| case AArch64::STRQui: | |
| case AArch64::STURQi: | |
| return AArch64::STRQpost; | |
| case AArch64::STRBBui: | |
| return AArch64::STRBBpost; | |
| case AArch64::STRHHui: | |
| return AArch64::STRHHpost; | |
| case AArch64::STRWui: | |
| case AArch64::STURWi: | |
| return AArch64::STRWpost; | |
| case AArch64::STRXui: | |
| case AArch64::STURXi: | |
| return AArch64::STRXpost; | |
| case AArch64::LDRSui: | |
| case AArch64::LDURSi: | |
| return AArch64::LDRSpost; | |
| case AArch64::LDRDui: | |
| case AArch64::LDURDi: | |
| return AArch64::LDRDpost; | |
| case AArch64::LDRQui: | |
| case AArch64::LDURQi: | |
| return AArch64::LDRQpost; | |
| case AArch64::LDRBBui: | |
| return AArch64::LDRBBpost; | |
| case AArch64::LDRHHui: | |
| return AArch64::LDRHHpost; | |
| case AArch64::LDRWui: | |
| case AArch64::LDURWi: | |
| return AArch64::LDRWpost; | |
| case AArch64::LDRXui: | |
| case AArch64::LDURXi: | |
| return AArch64::LDRXpost; | |
| case AArch64::LDRSWui: | |
| return AArch64::LDRSWpost; | |
| case AArch64::LDPSi: | |
| return AArch64::LDPSpost; | |
| case AArch64::LDPSWi: | |
| return AArch64::LDPSWpost; | |
| case AArch64::LDPDi: | |
| return AArch64::LDPDpost; | |
| case AArch64::LDPQi: | |
| return AArch64::LDPQpost; | |
| case AArch64::LDPWi: | |
| return AArch64::LDPWpost; | |
| case AArch64::LDPXi: | |
| return AArch64::LDPXpost; | |
| case AArch64::STPSi: | |
| return AArch64::STPSpost; | |
| case AArch64::STPDi: | |
| return AArch64::STPDpost; | |
| case AArch64::STPQi: | |
| return AArch64::STPQpost; | |
| case AArch64::STPWi: | |
| return AArch64::STPWpost; | |
| case AArch64::STPXi: | |
| return AArch64::STPXpost; | |
| } | |
| } | |
| static bool isPairedLdSt(const MachineInstr &MI) { | |
| switch (MI.getOpcode()) { | |
| default: | |
| return false; | |
| case AArch64::LDPSi: | |
| case AArch64::LDPSWi: | |
| case AArch64::LDPDi: | |
| case AArch64::LDPQi: | |
| case AArch64::LDPWi: | |
| case AArch64::LDPXi: | |
| case AArch64::STPSi: | |
| case AArch64::STPDi: | |
| case AArch64::STPQi: | |
| case AArch64::STPWi: | |
| case AArch64::STPXi: | |
| return true; | |
| } | |
| } | |
| static const MachineOperand &getLdStRegOp(const MachineInstr &MI, | |
| unsigned PairedRegOp = 0) { | |
| assert(PairedRegOp < 2 && "Unexpected register operand idx."); | |
| unsigned Idx = isPairedLdSt(MI) ? PairedRegOp : 0; | |
| return MI.getOperand(Idx); | |
| } | |
| static const MachineOperand &getLdStBaseOp(const MachineInstr &MI) { | |
| unsigned Idx = isPairedLdSt(MI) ? 2 : 1; | |
| return MI.getOperand(Idx); | |
| } | |
| static const MachineOperand &getLdStOffsetOp(const MachineInstr &MI) { | |
| unsigned Idx = isPairedLdSt(MI) ? 3 : 2; | |
| return MI.getOperand(Idx); | |
| } | |
| static bool isLdOffsetInRangeOfSt(MachineInstr &LoadInst, | |
| MachineInstr &StoreInst, | |
| const AArch64InstrInfo *TII) { | |
| assert(isMatchingStore(LoadInst, StoreInst) && "Expect only matched ld/st."); | |
| int LoadSize = getMemScale(LoadInst); | |
| int StoreSize = getMemScale(StoreInst); | |
| int UnscaledStOffset = TII->isUnscaledLdSt(StoreInst) | |
| ? getLdStOffsetOp(StoreInst).getImm() | |
| : getLdStOffsetOp(StoreInst).getImm() * StoreSize; | |
| int UnscaledLdOffset = TII->isUnscaledLdSt(LoadInst) | |
| ? getLdStOffsetOp(LoadInst).getImm() | |
| : getLdStOffsetOp(LoadInst).getImm() * LoadSize; | |
| return (UnscaledStOffset <= UnscaledLdOffset) && | |
| (UnscaledLdOffset + LoadSize <= (UnscaledStOffset + StoreSize)); | |
| } | |
| static bool isPromotableZeroStoreInst(MachineInstr &MI) { | |
| unsigned Opc = MI.getOpcode(); | |
| return (Opc == AArch64::STRWui || Opc == AArch64::STURWi || | |
| isNarrowStore(Opc)) && | |
| getLdStRegOp(MI).getReg() == AArch64::WZR; | |
| } | |
| static bool isPromotableLoadFromStore(MachineInstr &MI) { | |
| switch (MI.getOpcode()) { | |
| default: | |
| return false; | |
| // Scaled instructions. | |
| case AArch64::LDRBBui: | |
| case AArch64::LDRHHui: | |
| case AArch64::LDRWui: | |
| case AArch64::LDRXui: | |
| // Unscaled instructions. | |
| case AArch64::LDURBBi: | |
| case AArch64::LDURHHi: | |
| case AArch64::LDURWi: | |
| case AArch64::LDURXi: | |
| return true; | |
| } | |
| } | |
| static bool isMergeableLdStUpdate(MachineInstr &MI) { | |
| unsigned Opc = MI.getOpcode(); | |
| switch (Opc) { | |
| default: | |
| return false; | |
| // Scaled instructions. | |
| case AArch64::STRSui: | |
| case AArch64::STRDui: | |
| case AArch64::STRQui: | |
| case AArch64::STRXui: | |
| case AArch64::STRWui: | |
| case AArch64::STRHHui: | |
| case AArch64::STRBBui: | |
| case AArch64::LDRSui: | |
| case AArch64::LDRDui: | |
| case AArch64::LDRQui: | |
| case AArch64::LDRXui: | |
| case AArch64::LDRWui: | |
| case AArch64::LDRHHui: | |
| case AArch64::LDRBBui: | |
| // Unscaled instructions. | |
| case AArch64::STURSi: | |
| case AArch64::STURDi: | |
| case AArch64::STURQi: | |
| case AArch64::STURWi: | |
| case AArch64::STURXi: | |
| case AArch64::LDURSi: | |
| case AArch64::LDURDi: | |
| case AArch64::LDURQi: | |
| case AArch64::LDURWi: | |
| case AArch64::LDURXi: | |
| // Paired instructions. | |
| case AArch64::LDPSi: | |
| case AArch64::LDPSWi: | |
| case AArch64::LDPDi: | |
| case AArch64::LDPQi: | |
| case AArch64::LDPWi: | |
| case AArch64::LDPXi: | |
| case AArch64::STPSi: | |
| case AArch64::STPDi: | |
| case AArch64::STPQi: | |
| case AArch64::STPWi: | |
| case AArch64::STPXi: | |
| // Make sure this is a reg+imm (as opposed to an address reloc). | |
| if (!getLdStOffsetOp(MI).isImm()) | |
| return false; | |
| return true; | |
| } | |
| } | |
| MachineBasicBlock::iterator | |
| AArch64LoadStoreOpt::mergeNarrowZeroStores(MachineBasicBlock::iterator I, | |
| MachineBasicBlock::iterator MergeMI, | |
| const LdStPairFlags &Flags) { | |
| assert(isPromotableZeroStoreInst(*I) && isPromotableZeroStoreInst(*MergeMI) && | |
| "Expected promotable zero stores."); | |
| MachineBasicBlock::iterator NextI = I; | |
| ++NextI; | |
| // If NextI is the second of the two instructions to be merged, we need | |
| // to skip one further. Either way we merge will invalidate the iterator, | |
| // and we don't need to scan the new instruction, as it's a pairwise | |
| // instruction, which we're not considering for further action anyway. | |
| if (NextI == MergeMI) | |
| ++NextI; | |
| unsigned Opc = I->getOpcode(); | |
| bool IsScaled = !TII->isUnscaledLdSt(Opc); | |
| int OffsetStride = IsScaled ? 1 : getMemScale(*I); | |
| bool MergeForward = Flags.getMergeForward(); | |
| // Insert our new paired instruction after whichever of the paired | |
| // instructions MergeForward indicates. | |
| MachineBasicBlock::iterator InsertionPoint = MergeForward ? MergeMI : I; | |
| // Also based on MergeForward is from where we copy the base register operand | |
| // so we get the flags compatible with the input code. | |
| const MachineOperand &BaseRegOp = | |
| MergeForward ? getLdStBaseOp(*MergeMI) : getLdStBaseOp(*I); | |
| // Which register is Rt and which is Rt2 depends on the offset order. | |
| MachineInstr *RtMI; | |
| if (getLdStOffsetOp(*I).getImm() == | |
| getLdStOffsetOp(*MergeMI).getImm() + OffsetStride) | |
| RtMI = &*MergeMI; | |
| else | |
| RtMI = &*I; | |
| int OffsetImm = getLdStOffsetOp(*RtMI).getImm(); | |
| // Change the scaled offset from small to large type. | |
| if (IsScaled) { | |
| assert(((OffsetImm & 1) == 0) && "Unexpected offset to merge"); | |
| OffsetImm /= 2; | |
| } | |
| // Construct the new instruction. | |
| DebugLoc DL = I->getDebugLoc(); | |
| MachineBasicBlock *MBB = I->getParent(); | |
| MachineInstrBuilder MIB; | |
| MIB = BuildMI(*MBB, InsertionPoint, DL, TII->get(getMatchingWideOpcode(Opc))) | |
| .addReg(isNarrowStore(Opc) ? AArch64::WZR : AArch64::XZR) | |
| .add(BaseRegOp) | |
| .addImm(OffsetImm) | |
| .setMemRefs(I->mergeMemRefsWith(*MergeMI)) | |
| .setMIFlags(I->mergeFlagsWith(*MergeMI)); | |
| (void)MIB; | |
| LLVM_DEBUG(dbgs() << "Creating wider store. Replacing instructions:\n "); | |
| LLVM_DEBUG(I->print(dbgs())); | |
| LLVM_DEBUG(dbgs() << " "); | |
| LLVM_DEBUG(MergeMI->print(dbgs())); | |
| LLVM_DEBUG(dbgs() << " with instruction:\n "); | |
| LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs())); | |
| LLVM_DEBUG(dbgs() << "\n"); | |
| // Erase the old instructions. | |
| I->eraseFromParent(); | |
| MergeMI->eraseFromParent(); | |
| return NextI; | |
| } | |
| MachineBasicBlock::iterator | |
| AArch64LoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I, | |
| MachineBasicBlock::iterator Paired, | |
| const LdStPairFlags &Flags) { | |
| MachineBasicBlock::iterator NextI = I; | |
| ++NextI; | |
| // If NextI is the second of the two instructions to be merged, we need | |
| // to skip one further. Either way we merge will invalidate the iterator, | |
| // and we don't need to scan the new instruction, as it's a pairwise | |
| // instruction, which we're not considering for further action anyway. | |
| if (NextI == Paired) | |
| ++NextI; | |
| int SExtIdx = Flags.getSExtIdx(); | |
| unsigned Opc = | |
| SExtIdx == -1 ? I->getOpcode() : getMatchingNonSExtOpcode(I->getOpcode()); | |
| bool IsUnscaled = TII->isUnscaledLdSt(Opc); | |
| int OffsetStride = IsUnscaled ? getMemScale(*I) : 1; | |
| bool MergeForward = Flags.getMergeForward(); | |
| // Insert our new paired instruction after whichever of the paired | |
| // instructions MergeForward indicates. | |
| MachineBasicBlock::iterator InsertionPoint = MergeForward ? Paired : I; | |
| // Also based on MergeForward is from where we copy the base register operand | |
| // so we get the flags compatible with the input code. | |
| const MachineOperand &BaseRegOp = | |
| MergeForward ? getLdStBaseOp(*Paired) : getLdStBaseOp(*I); | |
| int Offset = getLdStOffsetOp(*I).getImm(); | |
| int PairedOffset = getLdStOffsetOp(*Paired).getImm(); | |
| bool PairedIsUnscaled = TII->isUnscaledLdSt(Paired->getOpcode()); | |
| if (IsUnscaled != PairedIsUnscaled) { | |
| // We're trying to pair instructions that differ in how they are scaled. If | |
| // I is scaled then scale the offset of Paired accordingly. Otherwise, do | |
| // the opposite (i.e., make Paired's offset unscaled). | |
| int MemSize = getMemScale(*Paired); | |
| if (PairedIsUnscaled) { | |
| // If the unscaled offset isn't a multiple of the MemSize, we can't | |
| // pair the operations together. | |
| assert(!(PairedOffset % getMemScale(*Paired)) && | |
| "Offset should be a multiple of the stride!"); | |
| PairedOffset /= MemSize; | |
| } else { | |
| PairedOffset *= MemSize; | |
| } | |
| } | |
| // Which register is Rt and which is Rt2 depends on the offset order. | |
| MachineInstr *RtMI, *Rt2MI; | |
| if (Offset == PairedOffset + OffsetStride) { | |
| RtMI = &*Paired; | |
| Rt2MI = &*I; | |
| // Here we swapped the assumption made for SExtIdx. | |
| // I.e., we turn ldp I, Paired into ldp Paired, I. | |
| // Update the index accordingly. | |
| if (SExtIdx != -1) | |
| SExtIdx = (SExtIdx + 1) % 2; | |
| } else { | |
| RtMI = &*I; | |
| Rt2MI = &*Paired; | |
| } | |
| int OffsetImm = getLdStOffsetOp(*RtMI).getImm(); | |
| // Scale the immediate offset, if necessary. | |
| if (TII->isUnscaledLdSt(RtMI->getOpcode())) { | |
| assert(!(OffsetImm % getMemScale(*RtMI)) && | |
| "Unscaled offset cannot be scaled."); | |
| OffsetImm /= getMemScale(*RtMI); | |
| } | |
| // Construct the new instruction. | |
| MachineInstrBuilder MIB; | |
| DebugLoc DL = I->getDebugLoc(); | |
| MachineBasicBlock *MBB = I->getParent(); | |
| MachineOperand RegOp0 = getLdStRegOp(*RtMI); | |
| MachineOperand RegOp1 = getLdStRegOp(*Rt2MI); | |
| // Kill flags may become invalid when moving stores for pairing. | |
| if (RegOp0.isUse()) { | |
| if (!MergeForward) { | |
| // Clear kill flags on store if moving upwards. Example: | |
| // STRWui %w0, ... | |
| // USE %w1 | |
| // STRWui kill %w1 ; need to clear kill flag when moving STRWui upwards | |
| RegOp0.setIsKill(false); | |
| RegOp1.setIsKill(false); | |
| } else { | |
| // Clear kill flags of the first stores register. Example: | |
| // STRWui %w1, ... | |
| // USE kill %w1 ; need to clear kill flag when moving STRWui downwards | |
| // STRW %w0 | |
| unsigned Reg = getLdStRegOp(*I).getReg(); | |
| for (MachineInstr &MI : make_range(std::next(I), Paired)) | |
| MI.clearRegisterKills(Reg, TRI); | |
| } | |
| } | |
| MIB = BuildMI(*MBB, InsertionPoint, DL, TII->get(getMatchingPairOpcode(Opc))) | |
| .add(RegOp0) | |
| .add(RegOp1) | |
| .add(BaseRegOp) | |
| .addImm(OffsetImm) | |
| .setMemRefs(I->mergeMemRefsWith(*Paired)) | |
| .setMIFlags(I->mergeFlagsWith(*Paired)); | |
| (void)MIB; | |
| LLVM_DEBUG( | |
| dbgs() << "Creating pair load/store. Replacing instructions:\n "); | |
| LLVM_DEBUG(I->print(dbgs())); | |
| LLVM_DEBUG(dbgs() << " "); | |
| LLVM_DEBUG(Paired->print(dbgs())); | |
| LLVM_DEBUG(dbgs() << " with instruction:\n "); | |
| if (SExtIdx != -1) { | |
| // Generate the sign extension for the proper result of the ldp. | |
| // I.e., with X1, that would be: | |
| // %w1 = KILL %w1, implicit-def %x1 | |
| // %x1 = SBFMXri killed %x1, 0, 31 | |
| MachineOperand &DstMO = MIB->getOperand(SExtIdx); | |
| // Right now, DstMO has the extended register, since it comes from an | |
| // extended opcode. | |
| unsigned DstRegX = DstMO.getReg(); | |
| // Get the W variant of that register. | |
| unsigned DstRegW = TRI->getSubReg(DstRegX, AArch64::sub_32); | |
| // Update the result of LDP to use the W instead of the X variant. | |
| DstMO.setReg(DstRegW); | |
| LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs())); | |
| LLVM_DEBUG(dbgs() << "\n"); | |
| // Make the machine verifier happy by providing a definition for | |
| // the X register. | |
| // Insert this definition right after the generated LDP, i.e., before | |
| // InsertionPoint. | |
| MachineInstrBuilder MIBKill = | |
| BuildMI(*MBB, InsertionPoint, DL, TII->get(TargetOpcode::KILL), DstRegW) | |
| .addReg(DstRegW) | |
| .addReg(DstRegX, RegState::Define); | |
| MIBKill->getOperand(2).setImplicit(); | |
| // Create the sign extension. | |
| MachineInstrBuilder MIBSXTW = | |
| BuildMI(*MBB, InsertionPoint, DL, TII->get(AArch64::SBFMXri), DstRegX) | |
| .addReg(DstRegX) | |
| .addImm(0) | |
| .addImm(31); | |
| (void)MIBSXTW; | |
| LLVM_DEBUG(dbgs() << " Extend operand:\n "); | |
| LLVM_DEBUG(((MachineInstr *)MIBSXTW)->print(dbgs())); | |
| } else { | |
| LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs())); | |
| } | |
| LLVM_DEBUG(dbgs() << "\n"); | |
| // Erase the old instructions. | |
| I->eraseFromParent(); | |
| Paired->eraseFromParent(); | |
| return NextI; | |
| } | |
| MachineBasicBlock::iterator | |
| AArch64LoadStoreOpt::promoteLoadFromStore(MachineBasicBlock::iterator LoadI, | |
| MachineBasicBlock::iterator StoreI) { | |
| MachineBasicBlock::iterator NextI = LoadI; | |
| ++NextI; | |
| int LoadSize = getMemScale(*LoadI); | |
| int StoreSize = getMemScale(*StoreI); | |
| unsigned LdRt = getLdStRegOp(*LoadI).getReg(); | |
| const MachineOperand &StMO = getLdStRegOp(*StoreI); | |
| unsigned StRt = getLdStRegOp(*StoreI).getReg(); | |
| bool IsStoreXReg = TRI->getRegClass(AArch64::GPR64RegClassID)->contains(StRt); | |
| assert((IsStoreXReg || | |
| TRI->getRegClass(AArch64::GPR32RegClassID)->contains(StRt)) && | |
| "Unexpected RegClass"); | |
| MachineInstr *BitExtMI; | |
| if (LoadSize == StoreSize && (LoadSize == 4 || LoadSize == 8)) { | |
| // Remove the load, if the destination register of the loads is the same | |
| // register for stored value. | |
| if (StRt == LdRt && LoadSize == 8) { | |
| for (MachineInstr &MI : make_range(StoreI->getIterator(), | |
| LoadI->getIterator())) { | |
| if (MI.killsRegister(StRt, TRI)) { | |
| MI.clearRegisterKills(StRt, TRI); | |
| break; | |
| } | |
| } | |
| LLVM_DEBUG(dbgs() << "Remove load instruction:\n "); | |
| LLVM_DEBUG(LoadI->print(dbgs())); | |
| LLVM_DEBUG(dbgs() << "\n"); | |
| LoadI->eraseFromParent(); | |
| return NextI; | |
| } | |
| // Replace the load with a mov if the load and store are in the same size. | |
| BitExtMI = | |
| BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(), | |
| TII->get(IsStoreXReg ? AArch64::ORRXrs : AArch64::ORRWrs), LdRt) | |
| .addReg(IsStoreXReg ? AArch64::XZR : AArch64::WZR) | |
| .add(StMO) | |
| .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0)) | |
| .setMIFlags(LoadI->getFlags()); | |
| } else { | |
| // FIXME: Currently we disable this transformation in big-endian targets as | |
| // performance and correctness are verified only in little-endian. | |
| if (!Subtarget->isLittleEndian()) | |
| return NextI; | |
| bool IsUnscaled = TII->isUnscaledLdSt(*LoadI); | |
| assert(IsUnscaled == TII->isUnscaledLdSt(*StoreI) && | |
| "Unsupported ld/st match"); | |
| assert(LoadSize <= StoreSize && "Invalid load size"); | |
| int UnscaledLdOffset = IsUnscaled | |
| ? getLdStOffsetOp(*LoadI).getImm() | |
| : getLdStOffsetOp(*LoadI).getImm() * LoadSize; | |
| int UnscaledStOffset = IsUnscaled | |
| ? getLdStOffsetOp(*StoreI).getImm() | |
| : getLdStOffsetOp(*StoreI).getImm() * StoreSize; | |
| int Width = LoadSize * 8; | |
| int Immr = 8 * (UnscaledLdOffset - UnscaledStOffset); | |
| int Imms = Immr + Width - 1; | |
| unsigned DestReg = IsStoreXReg | |
| ? TRI->getMatchingSuperReg(LdRt, AArch64::sub_32, | |
| &AArch64::GPR64RegClass) | |
| : LdRt; | |
| assert((UnscaledLdOffset >= UnscaledStOffset && | |
| (UnscaledLdOffset + LoadSize) <= UnscaledStOffset + StoreSize) && | |
| "Invalid offset"); | |
| Immr = 8 * (UnscaledLdOffset - UnscaledStOffset); | |
| Imms = Immr + Width - 1; | |
| if (UnscaledLdOffset == UnscaledStOffset) { | |
| uint32_t AndMaskEncoded = ((IsStoreXReg ? 1 : 0) << 12) // N | |
| | ((Immr) << 6) // immr | |
| | ((Imms) << 0) // imms | |
| ; | |
| BitExtMI = | |
| BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(), | |
| TII->get(IsStoreXReg ? AArch64::ANDXri : AArch64::ANDWri), | |
| DestReg) | |
| .add(StMO) | |
| .addImm(AndMaskEncoded) | |
| .setMIFlags(LoadI->getFlags()); | |
| } else { | |
| BitExtMI = | |
| BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(), | |
| TII->get(IsStoreXReg ? AArch64::UBFMXri : AArch64::UBFMWri), | |
| DestReg) | |
| .add(StMO) | |
| .addImm(Immr) | |
| .addImm(Imms) | |
| .setMIFlags(LoadI->getFlags()); | |
| } | |
| } | |
| // Clear kill flags between store and load. | |
| for (MachineInstr &MI : make_range(StoreI->getIterator(), | |
| BitExtMI->getIterator())) | |
| if (MI.killsRegister(StRt, TRI)) { | |
| MI.clearRegisterKills(StRt, TRI); | |
| break; | |
| } | |
| LLVM_DEBUG(dbgs() << "Promoting load by replacing :\n "); | |
| LLVM_DEBUG(StoreI->print(dbgs())); | |
| LLVM_DEBUG(dbgs() << " "); | |
| LLVM_DEBUG(LoadI->print(dbgs())); | |
| LLVM_DEBUG(dbgs() << " with instructions:\n "); | |
| LLVM_DEBUG(StoreI->print(dbgs())); | |
| LLVM_DEBUG(dbgs() << " "); | |
| LLVM_DEBUG((BitExtMI)->print(dbgs())); | |
| LLVM_DEBUG(dbgs() << "\n"); | |
| // Erase the old instructions. | |
| LoadI->eraseFromParent(); | |
| return NextI; | |
| } | |
| static bool inBoundsForPair(bool IsUnscaled, int Offset, int OffsetStride) { | |
| // Convert the byte-offset used by unscaled into an "element" offset used | |
| // by the scaled pair load/store instructions. | |
| if (IsUnscaled) { | |
| // If the byte-offset isn't a multiple of the stride, there's no point | |
| // trying to match it. | |
| if (Offset % OffsetStride) | |
| return false; | |
| Offset /= OffsetStride; | |
| } | |
| return Offset <= 63 && Offset >= -64; | |
| } | |
| // Do alignment, specialized to power of 2 and for signed ints, | |
| // avoiding having to do a C-style cast from uint_64t to int when | |
| // using alignTo from include/llvm/Support/MathExtras.h. | |
| // FIXME: Move this function to include/MathExtras.h? | |
| static int alignTo(int Num, int PowOf2) { | |
| return (Num + PowOf2 - 1) & ~(PowOf2 - 1); | |
| } | |
| static bool mayAlias(MachineInstr &MIa, MachineInstr &MIb, | |
| AliasAnalysis *AA) { | |
| // One of the instructions must modify memory. | |
| if (!MIa.mayStore() && !MIb.mayStore()) | |
| return false; | |
| // Both instructions must be memory operations. | |
| if (!MIa.mayLoadOrStore() && !MIb.mayLoadOrStore()) | |
| return false; | |
| return MIa.mayAlias(AA, MIb, /*UseTBAA*/false); | |
| } | |
| static bool mayAlias(MachineInstr &MIa, | |
| SmallVectorImpl<MachineInstr *> &MemInsns, | |
| AliasAnalysis *AA) { | |
| for (MachineInstr *MIb : MemInsns) | |
| if (mayAlias(MIa, *MIb, AA)) | |
| return true; | |
| return false; | |
| } | |
| bool AArch64LoadStoreOpt::findMatchingStore( | |
| MachineBasicBlock::iterator I, unsigned Limit, | |
| MachineBasicBlock::iterator &StoreI) { | |
| MachineBasicBlock::iterator B = I->getParent()->begin(); | |
| MachineBasicBlock::iterator MBBI = I; | |
| MachineInstr &LoadMI = *I; | |
| unsigned BaseReg = getLdStBaseOp(LoadMI).getReg(); | |
| // If the load is the first instruction in the block, there's obviously | |
| // not any matching store. | |
| if (MBBI == B) | |
| return false; | |
| // Track which register units have been modified and used between the first | |
| // insn and the second insn. | |
| ModifiedRegUnits.clear(); | |
| UsedRegUnits.clear(); | |
| unsigned Count = 0; | |
| do { | |
| --MBBI; | |
| MachineInstr &MI = *MBBI; | |
| // Don't count transient instructions towards the search limit since there | |
| // may be different numbers of them if e.g. debug information is present. | |
| if (!MI.isTransient()) | |
| ++Count; | |
| // If the load instruction reads directly from the address to which the | |
| // store instruction writes and the stored value is not modified, we can | |
| // promote the load. Since we do not handle stores with pre-/post-index, | |
| // it's unnecessary to check if BaseReg is modified by the store itself. | |
| if (MI.mayStore() && isMatchingStore(LoadMI, MI) && | |
| BaseReg == getLdStBaseOp(MI).getReg() && | |
| isLdOffsetInRangeOfSt(LoadMI, MI, TII) && | |
| ModifiedRegUnits.available(getLdStRegOp(MI).getReg())) { | |
| StoreI = MBBI; | |
| return true; | |
| } | |
| if (MI.isCall()) | |
| return false; | |
| // Update modified / uses register units. | |
| LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI); | |
| // Otherwise, if the base register is modified, we have no match, so | |
| // return early. | |
| if (!ModifiedRegUnits.available(BaseReg)) | |
| return false; | |
| // If we encounter a store aliased with the load, return early. | |
| if (MI.mayStore() && mayAlias(LoadMI, MI, AA)) | |
| return false; | |
| } while (MBBI != B && Count < Limit); | |
| return false; | |
| } | |
| // Returns true if FirstMI and MI are candidates for merging or pairing. | |
| // Otherwise, returns false. | |
| static bool areCandidatesToMergeOrPair(MachineInstr &FirstMI, MachineInstr &MI, | |
| LdStPairFlags &Flags, | |
| const AArch64InstrInfo *TII) { | |
| // If this is volatile or if pairing is suppressed, not a candidate. | |
| if (MI.hasOrderedMemoryRef() || TII->isLdStPairSuppressed(MI)) | |
| return false; | |
| // We should have already checked FirstMI for pair suppression and volatility. | |
| assert(!FirstMI.hasOrderedMemoryRef() && | |
| !TII->isLdStPairSuppressed(FirstMI) && | |
| "FirstMI shouldn't get here if either of these checks are true."); | |
| unsigned OpcA = FirstMI.getOpcode(); | |
| unsigned OpcB = MI.getOpcode(); | |
| // Opcodes match: nothing more to check. | |
| if (OpcA == OpcB) | |
| return true; | |
| // Try to match a sign-extended load/store with a zero-extended load/store. | |
| bool IsValidLdStrOpc, PairIsValidLdStrOpc; | |
| unsigned NonSExtOpc = getMatchingNonSExtOpcode(OpcA, &IsValidLdStrOpc); | |
| assert(IsValidLdStrOpc && | |
| "Given Opc should be a Load or Store with an immediate"); | |
| // OpcA will be the first instruction in the pair. | |
| if (NonSExtOpc == getMatchingNonSExtOpcode(OpcB, &PairIsValidLdStrOpc)) { | |
| Flags.setSExtIdx(NonSExtOpc == (unsigned)OpcA ? 1 : 0); | |
| return true; | |
| } | |
| // If the second instruction isn't even a mergable/pairable load/store, bail | |
| // out. | |
| if (!PairIsValidLdStrOpc) | |
| return false; | |
| // FIXME: We don't support merging narrow stores with mixed scaled/unscaled | |
| // offsets. | |
| if (isNarrowStore(OpcA) || isNarrowStore(OpcB)) | |
| return false; | |
| // Try to match an unscaled load/store with a scaled load/store. | |
| return TII->isUnscaledLdSt(OpcA) != TII->isUnscaledLdSt(OpcB) && | |
| getMatchingPairOpcode(OpcA) == getMatchingPairOpcode(OpcB); | |
| // FIXME: Can we also match a mixed sext/zext unscaled/scaled pair? | |
| } | |
| /// Scan the instructions looking for a load/store that can be combined with the | |
| /// current instruction into a wider equivalent or a load/store pair. | |
| MachineBasicBlock::iterator | |
| AArch64LoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I, | |
| LdStPairFlags &Flags, unsigned Limit, | |
| bool FindNarrowMerge) { | |
| MachineBasicBlock::iterator E = I->getParent()->end(); | |
| MachineBasicBlock::iterator MBBI = I; | |
| MachineInstr &FirstMI = *I; | |
| ++MBBI; | |
| bool MayLoad = FirstMI.mayLoad(); | |
| bool IsUnscaled = TII->isUnscaledLdSt(FirstMI); | |
| unsigned Reg = getLdStRegOp(FirstMI).getReg(); | |
| unsigned BaseReg = getLdStBaseOp(FirstMI).getReg(); | |
| int Offset = getLdStOffsetOp(FirstMI).getImm(); | |
| int OffsetStride = IsUnscaled ? getMemScale(FirstMI) : 1; | |
| bool IsPromotableZeroStore = isPromotableZeroStoreInst(FirstMI); | |
| // Track which register units have been modified and used between the first | |
| // insn (inclusive) and the second insn. | |
| ModifiedRegUnits.clear(); | |
| UsedRegUnits.clear(); | |
| // Remember any instructions that read/write memory between FirstMI and MI. | |
| SmallVector<MachineInstr *, 4> MemInsns; | |
| for (unsigned Count = 0; MBBI != E && Count < Limit; ++MBBI) { | |
| MachineInstr &MI = *MBBI; | |
| // Don't count transient instructions towards the search limit since there | |
| // may be different numbers of them if e.g. debug information is present. | |
| if (!MI.isTransient()) | |
| ++Count; | |
| Flags.setSExtIdx(-1); | |
| if (areCandidatesToMergeOrPair(FirstMI, MI, Flags, TII) && | |
| getLdStOffsetOp(MI).isImm()) { | |
| assert(MI.mayLoadOrStore() && "Expected memory operation."); | |
| // If we've found another instruction with the same opcode, check to see | |
| // if the base and offset are compatible with our starting instruction. | |
| // These instructions all have scaled immediate operands, so we just | |
| // check for +1/-1. Make sure to check the new instruction offset is | |
| // actually an immediate and not a symbolic reference destined for | |
| // a relocation. | |
| unsigned MIBaseReg = getLdStBaseOp(MI).getReg(); | |
| int MIOffset = getLdStOffsetOp(MI).getImm(); | |
| bool MIIsUnscaled = TII->isUnscaledLdSt(MI); | |
| if (IsUnscaled != MIIsUnscaled) { | |
| // We're trying to pair instructions that differ in how they are scaled. | |
| // If FirstMI is scaled then scale the offset of MI accordingly. | |
| // Otherwise, do the opposite (i.e., make MI's offset unscaled). | |
| int MemSize = getMemScale(MI); | |
| if (MIIsUnscaled) { | |
| // If the unscaled offset isn't a multiple of the MemSize, we can't | |
| // pair the operations together: bail and keep looking. | |
| if (MIOffset % MemSize) { | |
| LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, | |
| UsedRegUnits, TRI); | |
| MemInsns.push_back(&MI); | |
| continue; | |
| } | |
| MIOffset /= MemSize; | |
| } else { | |
| MIOffset *= MemSize; | |
| } | |
| } | |
| if (BaseReg == MIBaseReg && ((Offset == MIOffset + OffsetStride) || | |
| (Offset + OffsetStride == MIOffset))) { | |
| int MinOffset = Offset < MIOffset ? Offset : MIOffset; | |
| if (FindNarrowMerge) { | |
| // If the alignment requirements of the scaled wide load/store | |
| // instruction can't express the offset of the scaled narrow input, | |
| // bail and keep looking. For promotable zero stores, allow only when | |
| // the stored value is the same (i.e., WZR). | |
| if ((!IsUnscaled && alignTo(MinOffset, 2) != MinOffset) || | |
| (IsPromotableZeroStore && Reg != getLdStRegOp(MI).getReg())) { | |
| LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, | |
| UsedRegUnits, TRI); | |
| MemInsns.push_back(&MI); | |
| continue; | |
| } | |
| } else { | |
| // Pairwise instructions have a 7-bit signed offset field. Single | |
| // insns have a 12-bit unsigned offset field. If the resultant | |
| // immediate offset of merging these instructions is out of range for | |
| // a pairwise instruction, bail and keep looking. | |
| if (!inBoundsForPair(IsUnscaled, MinOffset, OffsetStride)) { | |
| LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, | |
| UsedRegUnits, TRI); | |
| MemInsns.push_back(&MI); | |
| continue; | |
| } | |
| // If the alignment requirements of the paired (scaled) instruction | |
| // can't express the offset of the unscaled input, bail and keep | |
| // looking. | |
| if (IsUnscaled && (alignTo(MinOffset, OffsetStride) != MinOffset)) { | |
| LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, | |
| UsedRegUnits, TRI); | |
| MemInsns.push_back(&MI); | |
| continue; | |
| } | |
| } | |
| // If the destination register of the loads is the same register, bail | |
| // and keep looking. A load-pair instruction with both destination | |
| // registers the same is UNPREDICTABLE and will result in an exception. | |
| if (MayLoad && Reg == getLdStRegOp(MI).getReg()) { | |
| LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, | |
| TRI); | |
| MemInsns.push_back(&MI); | |
| continue; | |
| } | |
| // If the Rt of the second instruction was not modified or used between | |
| // the two instructions and none of the instructions between the second | |
| // and first alias with the second, we can combine the second into the | |
| // first. | |
| if (ModifiedRegUnits.available(getLdStRegOp(MI).getReg()) && | |
| !(MI.mayLoad() && | |
| !UsedRegUnits.available(getLdStRegOp(MI).getReg())) && | |
| !mayAlias(MI, MemInsns, AA)) { | |
| Flags.setMergeForward(false); | |
| return MBBI; | |
| } | |
| // Likewise, if the Rt of the first instruction is not modified or used | |
| // between the two instructions and none of the instructions between the | |
| // first and the second alias with the first, we can combine the first | |
| // into the second. | |
| if (ModifiedRegUnits.available(getLdStRegOp(FirstMI).getReg()) && | |
| !(MayLoad && | |
| !UsedRegUnits.available(getLdStRegOp(FirstMI).getReg())) && | |
| !mayAlias(FirstMI, MemInsns, AA)) { | |
| Flags.setMergeForward(true); | |
| return MBBI; | |
| } | |
| // Unable to combine these instructions due to interference in between. | |
| // Keep looking. | |
| } | |
| } | |
| // If the instruction wasn't a matching load or store. Stop searching if we | |
| // encounter a call instruction that might modify memory. | |
| if (MI.isCall()) | |
| return E; | |
| // Update modified / uses register units. | |
| LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI); | |
| // Otherwise, if the base register is modified, we have no match, so | |
| // return early. | |
| if (!ModifiedRegUnits.available(BaseReg)) | |
| return E; | |
| // Update list of instructions that read/write memory. | |
| if (MI.mayLoadOrStore()) | |
| MemInsns.push_back(&MI); | |
| } | |
| return E; | |
| } | |
| MachineBasicBlock::iterator | |
| AArch64LoadStoreOpt::mergeUpdateInsn(MachineBasicBlock::iterator I, | |
| MachineBasicBlock::iterator Update, | |
| bool IsPreIdx) { | |
| assert((Update->getOpcode() == AArch64::ADDXri || | |
| Update->getOpcode() == AArch64::SUBXri) && | |
| "Unexpected base register update instruction to merge!"); | |
| MachineBasicBlock::iterator NextI = I; | |
| // Return the instruction following the merged instruction, which is | |
| // the instruction following our unmerged load. Unless that's the add/sub | |
| // instruction we're merging, in which case it's the one after that. | |
| if (++NextI == Update) | |
| ++NextI; | |
| int Value = Update->getOperand(2).getImm(); | |
| assert(AArch64_AM::getShiftValue(Update->getOperand(3).getImm()) == 0 && | |
| "Can't merge 1 << 12 offset into pre-/post-indexed load / store"); | |
| if (Update->getOpcode() == AArch64::SUBXri) | |
| Value = -Value; | |
| unsigned NewOpc = IsPreIdx ? getPreIndexedOpcode(I->getOpcode()) | |
| : getPostIndexedOpcode(I->getOpcode()); | |
| MachineInstrBuilder MIB; | |
| if (!isPairedLdSt(*I)) { | |
| // Non-paired instruction. | |
| MIB = BuildMI(*I->getParent(), I, I->getDebugLoc(), TII->get(NewOpc)) | |
| .add(getLdStRegOp(*Update)) | |
| .add(getLdStRegOp(*I)) | |
| .add(getLdStBaseOp(*I)) | |
| .addImm(Value) | |
| .setMemRefs(I->memoperands_begin(), I->memoperands_end()) | |
| .setMIFlags(I->mergeFlagsWith(*Update)); | |
| } else { | |
| // Paired instruction. | |
| int Scale = getMemScale(*I); | |
| MIB = BuildMI(*I->getParent(), I, I->getDebugLoc(), TII->get(NewOpc)) | |
| .add(getLdStRegOp(*Update)) | |
| .add(getLdStRegOp(*I, 0)) | |
| .add(getLdStRegOp(*I, 1)) | |
| .add(getLdStBaseOp(*I)) | |
| .addImm(Value / Scale) | |
| .setMemRefs(I->memoperands_begin(), I->memoperands_end()) | |
| .setMIFlags(I->mergeFlagsWith(*Update)); | |
| } | |
| (void)MIB; | |
| if (IsPreIdx) { | |
| ++NumPreFolded; | |
| LLVM_DEBUG(dbgs() << "Creating pre-indexed load/store."); | |
| } else { | |
| ++NumPostFolded; | |
| LLVM_DEBUG(dbgs() << "Creating post-indexed load/store."); | |
| } | |
| LLVM_DEBUG(dbgs() << " Replacing instructions:\n "); | |
| LLVM_DEBUG(I->print(dbgs())); | |
| LLVM_DEBUG(dbgs() << " "); | |
| LLVM_DEBUG(Update->print(dbgs())); | |
| LLVM_DEBUG(dbgs() << " with instruction:\n "); | |
| LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs())); | |
| LLVM_DEBUG(dbgs() << "\n"); | |
| // Erase the old instructions for the block. | |
| I->eraseFromParent(); | |
| Update->eraseFromParent(); | |
| return NextI; | |
| } | |
| bool AArch64LoadStoreOpt::isMatchingUpdateInsn(MachineInstr &MemMI, | |
| MachineInstr &MI, | |
| unsigned BaseReg, int Offset) { | |
| switch (MI.getOpcode()) { | |
| default: | |
| break; | |
| case AArch64::SUBXri: | |
| case AArch64::ADDXri: | |
| // Make sure it's a vanilla immediate operand, not a relocation or | |
| // anything else we can't handle. | |
| if (!MI.getOperand(2).isImm()) | |
| break; | |
| // Watch out for 1 << 12 shifted value. | |
| if (AArch64_AM::getShiftValue(MI.getOperand(3).getImm())) | |
| break; | |
| // The update instruction source and destination register must be the | |
| // same as the load/store base register. | |
| if (MI.getOperand(0).getReg() != BaseReg || | |
| MI.getOperand(1).getReg() != BaseReg) | |
| break; | |
| bool IsPairedInsn = isPairedLdSt(MemMI); | |
| int UpdateOffset = MI.getOperand(2).getImm(); | |
| if (MI.getOpcode() == AArch64::SUBXri) | |
| UpdateOffset = -UpdateOffset; | |
| // For non-paired load/store instructions, the immediate must fit in a | |
| // signed 9-bit integer. | |
| if (!IsPairedInsn && (UpdateOffset > 255 || UpdateOffset < -256)) | |
| break; | |
| // For paired load/store instructions, the immediate must be a multiple of | |
| // the scaling factor. The scaled offset must also fit into a signed 7-bit | |
| // integer. | |
| if (IsPairedInsn) { | |
| int Scale = getMemScale(MemMI); | |
| if (UpdateOffset % Scale != 0) | |
| break; | |
| int ScaledOffset = UpdateOffset / Scale; | |
| if (ScaledOffset > 63 || ScaledOffset < -64) | |
| break; | |
| } | |
| // If we have a non-zero Offset, we check that it matches the amount | |
| // we're adding to the register. | |
| if (!Offset || Offset == UpdateOffset) | |
| return true; | |
| break; | |
| } | |
| return false; | |
| } | |
| MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnForward( | |
| MachineBasicBlock::iterator I, int UnscaledOffset, unsigned Limit) { | |
| MachineBasicBlock::iterator E = I->getParent()->end(); | |
| MachineInstr &MemMI = *I; | |
| MachineBasicBlock::iterator MBBI = I; | |
| unsigned BaseReg = getLdStBaseOp(MemMI).getReg(); | |
| int MIUnscaledOffset = getLdStOffsetOp(MemMI).getImm() * getMemScale(MemMI); | |
| // Scan forward looking for post-index opportunities. Updating instructions | |
| // can't be formed if the memory instruction doesn't have the offset we're | |
| // looking for. | |
| if (MIUnscaledOffset != UnscaledOffset) | |
| return E; | |
| // If the base register overlaps a destination register, we can't | |
| // merge the update. | |
| bool IsPairedInsn = isPairedLdSt(MemMI); | |
| for (unsigned i = 0, e = IsPairedInsn ? 2 : 1; i != e; ++i) { | |
| unsigned DestReg = getLdStRegOp(MemMI, i).getReg(); | |
| if (DestReg == BaseReg || TRI->isSubRegister(BaseReg, DestReg)) | |
| return E; | |
| } | |
| // Track which register units have been modified and used between the first | |
| // insn (inclusive) and the second insn. | |
| ModifiedRegUnits.clear(); | |
| UsedRegUnits.clear(); | |
| ++MBBI; | |
| for (unsigned Count = 0; MBBI != E && Count < Limit; ++MBBI) { | |
| MachineInstr &MI = *MBBI; | |
| // Don't count transient instructions towards the search limit since there | |
| // may be different numbers of them if e.g. debug information is present. | |
| if (!MI.isTransient()) | |
| ++Count; | |
| // If we found a match, return it. | |
| if (isMatchingUpdateInsn(*I, MI, BaseReg, UnscaledOffset)) | |
| return MBBI; | |
| // Update the status of what the instruction clobbered and used. | |
| LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI); | |
| // Otherwise, if the base register is used or modified, we have no match, so | |
| // return early. | |
| if (!ModifiedRegUnits.available(BaseReg) || | |
| !UsedRegUnits.available(BaseReg)) | |
| return E; | |
| } | |
| return E; | |
| } | |
| MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnBackward( | |
| MachineBasicBlock::iterator I, unsigned Limit) { | |
| MachineBasicBlock::iterator B = I->getParent()->begin(); | |
| MachineBasicBlock::iterator E = I->getParent()->end(); | |
| MachineInstr &MemMI = *I; | |
| MachineBasicBlock::iterator MBBI = I; | |
| unsigned BaseReg = getLdStBaseOp(MemMI).getReg(); | |
| int Offset = getLdStOffsetOp(MemMI).getImm(); | |
| // If the load/store is the first instruction in the block, there's obviously | |
| // not any matching update. Ditto if the memory offset isn't zero. | |
| if (MBBI == B || Offset != 0) | |
| return E; | |
| // If the base register overlaps a destination register, we can't | |
| // merge the update. | |
| bool IsPairedInsn = isPairedLdSt(MemMI); | |
| for (unsigned i = 0, e = IsPairedInsn ? 2 : 1; i != e; ++i) { | |
| unsigned DestReg = getLdStRegOp(MemMI, i).getReg(); | |
| if (DestReg == BaseReg || TRI->isSubRegister(BaseReg, DestReg)) | |
| return E; | |
| } | |
| // Track which register units have been modified and used between the first | |
| // insn (inclusive) and the second insn. | |
| ModifiedRegUnits.clear(); | |
| UsedRegUnits.clear(); | |
| unsigned Count = 0; | |
| do { | |
| --MBBI; | |
| MachineInstr &MI = *MBBI; | |
| // Don't count transient instructions towards the search limit since there | |
| // may be different numbers of them if e.g. debug information is present. | |
| if (!MI.isTransient()) | |
| ++Count; | |
| // If we found a match, return it. | |
| if (isMatchingUpdateInsn(*I, MI, BaseReg, Offset)) | |
| return MBBI; | |
| // Update the status of what the instruction clobbered and used. | |
| LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI); | |
| // Otherwise, if the base register is used or modified, we have no match, so | |
| // return early. | |
| if (!ModifiedRegUnits.available(BaseReg) || | |
| !UsedRegUnits.available(BaseReg)) | |
| return E; | |
| } while (MBBI != B && Count < Limit); | |
| return E; | |
| } | |
| bool AArch64LoadStoreOpt::tryToPromoteLoadFromStore( | |
| MachineBasicBlock::iterator &MBBI) { | |
| MachineInstr &MI = *MBBI; | |
| // If this is a volatile load, don't mess with it. | |
| if (MI.hasOrderedMemoryRef()) | |
| return false; | |
| // Make sure this is a reg+imm. | |
| // FIXME: It is possible to extend it to handle reg+reg cases. | |
| if (!getLdStOffsetOp(MI).isImm()) | |
| return false; | |
| // Look backward up to LdStLimit instructions. | |
| MachineBasicBlock::iterator StoreI; | |
| if (findMatchingStore(MBBI, LdStLimit, StoreI)) { | |
| ++NumLoadsFromStoresPromoted; | |
| // Promote the load. Keeping the iterator straight is a | |
| // pain, so we let the merge routine tell us what the next instruction | |
| // is after it's done mucking about. | |
| MBBI = promoteLoadFromStore(MBBI, StoreI); | |
| return true; | |
| } | |
| return false; | |
| } | |
| // Merge adjacent zero stores into a wider store. | |
| bool AArch64LoadStoreOpt::tryToMergeZeroStInst( | |
| MachineBasicBlock::iterator &MBBI) { | |
| assert(isPromotableZeroStoreInst(*MBBI) && "Expected narrow store."); | |
| MachineInstr &MI = *MBBI; | |
| MachineBasicBlock::iterator E = MI.getParent()->end(); | |
| if (!TII->isCandidateToMergeOrPair(MI)) | |
| return false; | |
| // Look ahead up to LdStLimit instructions for a mergable instruction. | |
| LdStPairFlags Flags; | |
| MachineBasicBlock::iterator MergeMI = | |
| findMatchingInsn(MBBI, Flags, LdStLimit, /* FindNarrowMerge = */ true); | |
| if (MergeMI != E) { | |
| ++NumZeroStoresPromoted; | |
| // Keeping the iterator straight is a pain, so we let the merge routine tell | |
| // us what the next instruction is after it's done mucking about. | |
| MBBI = mergeNarrowZeroStores(MBBI, MergeMI, Flags); | |
| return true; | |
| } | |
| return false; | |
| } | |
| // Find loads and stores that can be merged into a single load or store pair | |
| // instruction. | |
| bool AArch64LoadStoreOpt::tryToPairLdStInst(MachineBasicBlock::iterator &MBBI) { | |
| MachineInstr &MI = *MBBI; | |
| MachineBasicBlock::iterator E = MI.getParent()->end(); | |
| if (!TII->isCandidateToMergeOrPair(MI)) | |
| return false; | |
| // Early exit if the offset is not possible to match. (6 bits of positive | |
| // range, plus allow an extra one in case we find a later insn that matches | |
| // with Offset-1) | |
| bool IsUnscaled = TII->isUnscaledLdSt(MI); | |
| int Offset = getLdStOffsetOp(MI).getImm(); | |
| int OffsetStride = IsUnscaled ? getMemScale(MI) : 1; | |
| // Allow one more for offset. | |
| if (Offset > 0) | |
| Offset -= OffsetStride; | |
| if (!inBoundsForPair(IsUnscaled, Offset, OffsetStride)) | |
| return false; | |
| // Look ahead up to LdStLimit instructions for a pairable instruction. | |
| LdStPairFlags Flags; | |
| MachineBasicBlock::iterator Paired = | |
| findMatchingInsn(MBBI, Flags, LdStLimit, /* FindNarrowMerge = */ false); | |
| if (Paired != E) { | |
| ++NumPairCreated; | |
| if (TII->isUnscaledLdSt(MI)) | |
| ++NumUnscaledPairCreated; | |
| // Keeping the iterator straight is a pain, so we let the merge routine tell | |
| // us what the next instruction is after it's done mucking about. | |
| MBBI = mergePairedInsns(MBBI, Paired, Flags); | |
| return true; | |
| } | |
| return false; | |
| } | |
| bool AArch64LoadStoreOpt::tryToMergeLdStUpdate | |
| (MachineBasicBlock::iterator &MBBI) { | |
| MachineInstr &MI = *MBBI; | |
| MachineBasicBlock::iterator E = MI.getParent()->end(); | |
| MachineBasicBlock::iterator Update; | |
| // Look forward to try to form a post-index instruction. For example, | |
| // ldr x0, [x20] | |
| // add x20, x20, #32 | |
| // merged into: | |
| // ldr x0, [x20], #32 | |
| Update = findMatchingUpdateInsnForward(MBBI, 0, UpdateLimit); | |
| if (Update != E) { | |
| // Merge the update into the ld/st. | |
| MBBI = mergeUpdateInsn(MBBI, Update, /*IsPreIdx=*/false); | |
| return true; | |
| } | |
| // Don't know how to handle unscaled pre/post-index versions below, so bail. | |
| if (TII->isUnscaledLdSt(MI.getOpcode())) | |
| return false; | |
| // Look back to try to find a pre-index instruction. For example, | |
| // add x0, x0, #8 | |
| // ldr x1, [x0] | |
| // merged into: | |
| // ldr x1, [x0, #8]! | |
| Update = findMatchingUpdateInsnBackward(MBBI, UpdateLimit); | |
| if (Update != E) { | |
| // Merge the update into the ld/st. | |
| MBBI = mergeUpdateInsn(MBBI, Update, /*IsPreIdx=*/true); | |
| return true; | |
| } | |
| // The immediate in the load/store is scaled by the size of the memory | |
| // operation. The immediate in the add we're looking for, | |
| // however, is not, so adjust here. | |
| int UnscaledOffset = getLdStOffsetOp(MI).getImm() * getMemScale(MI); | |
| // Look forward to try to find a post-index instruction. For example, | |
| // ldr x1, [x0, #64] | |
| // add x0, x0, #64 | |
| // merged into: | |
| // ldr x1, [x0, #64]! | |
| Update = findMatchingUpdateInsnForward(MBBI, UnscaledOffset, UpdateLimit); | |
| if (Update != E) { | |
| // Merge the update into the ld/st. | |
| MBBI = mergeUpdateInsn(MBBI, Update, /*IsPreIdx=*/true); | |
| return true; | |
| } | |
| return false; | |
| } | |
| bool AArch64LoadStoreOpt::optimizeBlock(MachineBasicBlock &MBB, | |
| bool EnableNarrowZeroStOpt) { | |
| bool Modified = false; | |
| // Four tranformations to do here: | |
| // 1) Find loads that directly read from stores and promote them by | |
| // replacing with mov instructions. If the store is wider than the load, | |
| // the load will be replaced with a bitfield extract. | |
| // e.g., | |
| // str w1, [x0, #4] | |
| // ldrh w2, [x0, #6] | |
| // ; becomes | |
| // str w1, [x0, #4] | |
| // lsr w2, w1, #16 | |
| for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end(); | |
| MBBI != E;) { | |
| if (isPromotableLoadFromStore(*MBBI) && tryToPromoteLoadFromStore(MBBI)) | |
| Modified = true; | |
| else | |
| ++MBBI; | |
| } | |
| // 2) Merge adjacent zero stores into a wider store. | |
| // e.g., | |
| // strh wzr, [x0] | |
| // strh wzr, [x0, #2] | |
| // ; becomes | |
| // str wzr, [x0] | |
| // e.g., | |
| // str wzr, [x0] | |
| // str wzr, [x0, #4] | |
| // ; becomes | |
| // str xzr, [x0] | |
| if (EnableNarrowZeroStOpt) | |
| for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end(); | |
| MBBI != E;) { | |
| if (isPromotableZeroStoreInst(*MBBI) && tryToMergeZeroStInst(MBBI)) | |
| Modified = true; | |
| else | |
| ++MBBI; | |
| } | |
| // 3) Find loads and stores that can be merged into a single load or store | |
| // pair instruction. | |
| // e.g., | |
| // ldr x0, [x2] | |
| // ldr x1, [x2, #8] | |
| // ; becomes | |
| // ldp x0, x1, [x2] | |
| for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end(); | |
| MBBI != E;) { | |
| if (TII->isPairableLdStInst(*MBBI) && tryToPairLdStInst(MBBI)) | |
| Modified = true; | |
| else | |
| ++MBBI; | |
| } | |
| // 4) Find base register updates that can be merged into the load or store | |
| // as a base-reg writeback. | |
| // e.g., | |
| // ldr x0, [x2] | |
| // add x2, x2, #4 | |
| // ; becomes | |
| // ldr x0, [x2], #4 | |
| for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end(); | |
| MBBI != E;) { | |
| if (isMergeableLdStUpdate(*MBBI) && tryToMergeLdStUpdate(MBBI)) | |
| Modified = true; | |
| else | |
| ++MBBI; | |
| } | |
| return Modified; | |
| } | |
| bool AArch64LoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) { | |
| if (skipFunction(Fn.getFunction())) | |
| return false; | |
| Subtarget = &static_cast<const AArch64Subtarget &>(Fn.getSubtarget()); | |
| TII = static_cast<const AArch64InstrInfo *>(Subtarget->getInstrInfo()); | |
| TRI = Subtarget->getRegisterInfo(); | |
| AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); | |
| // Resize the modified and used register unit trackers. We do this once | |
| // per function and then clear the register units each time we optimize a load | |
| // or store. | |
| ModifiedRegUnits.init(*TRI); | |
| UsedRegUnits.init(*TRI); | |
| bool Modified = false; | |
| bool enableNarrowZeroStOpt = !Subtarget->requiresStrictAlign(); | |
| for (auto &MBB : Fn) | |
| Modified |= optimizeBlock(MBB, enableNarrowZeroStOpt); | |
| return Modified; | |
| } | |
| // FIXME: Do we need/want a pre-alloc pass like ARM has to try to keep loads and | |
| // stores near one another? Note: The pre-RA instruction scheduler already has | |
| // hooks to try and schedule pairable loads/stores together to improve pairing | |
| // opportunities. Thus, pre-RA pairing pass may not be worth the effort. | |
| // FIXME: When pairing store instructions it's very possible for this pass to | |
| // hoist a store with a KILL marker above another use (without a KILL marker). | |
| // The resulting IR is invalid, but nothing uses the KILL markers after this | |
| // pass, so it's never caused a problem in practice. | |
| /// createAArch64LoadStoreOptimizationPass - returns an instance of the | |
| /// load / store optimization pass. | |
| FunctionPass *llvm::createAArch64LoadStoreOptimizationPass() { | |
| return new AArch64LoadStoreOpt(); | |
| } |