| //===-- SIFormMemoryClauses.cpp -------------------------------------------===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| // |
| /// \file |
| /// This pass creates bundles of SMEM and VMEM instructions forming memory |
| /// clauses if XNACK is enabled. Def operands of clauses are marked as early |
| /// clobber to make sure we will not override any source within a clause. |
| /// |
| //===----------------------------------------------------------------------===// |
| |
| #include "AMDGPU.h" |
| #include "AMDGPUSubtarget.h" |
| #include "GCNRegPressure.h" |
| #include "MCTargetDesc/AMDGPUMCTargetDesc.h" |
| #include "SIInstrInfo.h" |
| #include "SIMachineFunctionInfo.h" |
| #include "SIRegisterInfo.h" |
| #include "llvm/ADT/DenseMap.h" |
| #include "llvm/CodeGen/LiveIntervals.h" |
| #include "llvm/CodeGen/MachineFunctionPass.h" |
| #include "llvm/InitializePasses.h" |
| |
| using namespace llvm; |
| |
| #define DEBUG_TYPE "si-form-memory-clauses" |
| |
| // Clauses longer then 15 instructions would overflow one of the counters |
| // and stall. They can stall even earlier if there are outstanding counters. |
| static cl::opt<unsigned> |
| MaxClause("amdgpu-max-memory-clause", cl::Hidden, cl::init(15), |
| cl::desc("Maximum length of a memory clause, instructions")); |
| |
| namespace { |
| |
| class SIFormMemoryClauses : public MachineFunctionPass { |
| typedef DenseMap<unsigned, std::pair<unsigned, LaneBitmask>> RegUse; |
| |
| public: |
| static char ID; |
| |
| public: |
| SIFormMemoryClauses() : MachineFunctionPass(ID) { |
| initializeSIFormMemoryClausesPass(*PassRegistry::getPassRegistry()); |
| } |
| |
| bool runOnMachineFunction(MachineFunction &MF) override; |
| |
| StringRef getPassName() const override { |
| return "SI Form memory clauses"; |
| } |
| |
| void getAnalysisUsage(AnalysisUsage &AU) const override { |
| AU.addRequired<LiveIntervals>(); |
| AU.setPreservesAll(); |
| MachineFunctionPass::getAnalysisUsage(AU); |
| } |
| |
| private: |
| template <typename Callable> |
| void forAllLanes(unsigned Reg, LaneBitmask LaneMask, Callable Func) const; |
| |
| bool canBundle(const MachineInstr &MI, RegUse &Defs, RegUse &Uses) const; |
| bool checkPressure(const MachineInstr &MI, GCNDownwardRPTracker &RPT); |
| void collectRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses) const; |
| bool processRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses, |
| GCNDownwardRPTracker &RPT); |
| |
| const GCNSubtarget *ST; |
| const SIRegisterInfo *TRI; |
| const MachineRegisterInfo *MRI; |
| SIMachineFunctionInfo *MFI; |
| |
| unsigned LastRecordedOccupancy; |
| unsigned MaxVGPRs; |
| unsigned MaxSGPRs; |
| }; |
| |
| } // End anonymous namespace. |
| |
| INITIALIZE_PASS_BEGIN(SIFormMemoryClauses, DEBUG_TYPE, |
| "SI Form memory clauses", false, false) |
| INITIALIZE_PASS_DEPENDENCY(LiveIntervals) |
| INITIALIZE_PASS_END(SIFormMemoryClauses, DEBUG_TYPE, |
| "SI Form memory clauses", false, false) |
| |
| |
| char SIFormMemoryClauses::ID = 0; |
| |
| char &llvm::SIFormMemoryClausesID = SIFormMemoryClauses::ID; |
| |
| FunctionPass *llvm::createSIFormMemoryClausesPass() { |
| return new SIFormMemoryClauses(); |
| } |
| |
| static bool isVMEMClauseInst(const MachineInstr &MI) { |
| return SIInstrInfo::isFLAT(MI) || SIInstrInfo::isVMEM(MI); |
| } |
| |
| static bool isSMEMClauseInst(const MachineInstr &MI) { |
| return SIInstrInfo::isSMRD(MI); |
| } |
| |
| // There no sense to create store clauses, they do not define anything, |
| // thus there is nothing to set early-clobber. |
| static bool isValidClauseInst(const MachineInstr &MI, bool IsVMEMClause) { |
| if (MI.isDebugValue() || MI.isBundled()) |
| return false; |
| if (!MI.mayLoad() || MI.mayStore()) |
| return false; |
| if (AMDGPU::getAtomicNoRetOp(MI.getOpcode()) != -1 || |
| AMDGPU::getAtomicRetOp(MI.getOpcode()) != -1) |
| return false; |
| if (IsVMEMClause && !isVMEMClauseInst(MI)) |
| return false; |
| if (!IsVMEMClause && !isSMEMClauseInst(MI)) |
| return false; |
| // If this is a load instruction where the result has been coalesced with an operand, then we cannot clause it. |
| for (const MachineOperand &ResMO : MI.defs()) { |
| Register ResReg = ResMO.getReg(); |
| for (const MachineOperand &MO : MI.uses()) { |
| if (!MO.isReg() || MO.isDef()) |
| continue; |
| if (MO.getReg() == ResReg) |
| return false; |
| } |
| break; // Only check the first def. |
| } |
| return true; |
| } |
| |
| static unsigned getMopState(const MachineOperand &MO) { |
| unsigned S = 0; |
| if (MO.isImplicit()) |
| S |= RegState::Implicit; |
| if (MO.isDead()) |
| S |= RegState::Dead; |
| if (MO.isUndef()) |
| S |= RegState::Undef; |
| if (MO.isKill()) |
| S |= RegState::Kill; |
| if (MO.isEarlyClobber()) |
| S |= RegState::EarlyClobber; |
| if (Register::isPhysicalRegister(MO.getReg()) && MO.isRenamable()) |
| S |= RegState::Renamable; |
| return S; |
| } |
| |
| template <typename Callable> |
| void SIFormMemoryClauses::forAllLanes(unsigned Reg, LaneBitmask LaneMask, |
| Callable Func) const { |
| if (LaneMask.all() || Register::isPhysicalRegister(Reg) || |
| LaneMask == MRI->getMaxLaneMaskForVReg(Reg)) { |
| Func(0); |
| return; |
| } |
| |
| const TargetRegisterClass *RC = MRI->getRegClass(Reg); |
| unsigned E = TRI->getNumSubRegIndices(); |
| SmallVector<unsigned, AMDGPU::NUM_TARGET_SUBREGS> CoveringSubregs; |
| for (unsigned Idx = 1; Idx < E; ++Idx) { |
| // Is this index even compatible with the given class? |
| if (TRI->getSubClassWithSubReg(RC, Idx) != RC) |
| continue; |
| LaneBitmask SubRegMask = TRI->getSubRegIndexLaneMask(Idx); |
| // Early exit if we found a perfect match. |
| if (SubRegMask == LaneMask) { |
| Func(Idx); |
| return; |
| } |
| |
| if ((SubRegMask & ~LaneMask).any() || (SubRegMask & LaneMask).none()) |
| continue; |
| |
| CoveringSubregs.push_back(Idx); |
| } |
| |
| llvm::sort(CoveringSubregs, [this](unsigned A, unsigned B) { |
| LaneBitmask MaskA = TRI->getSubRegIndexLaneMask(A); |
| LaneBitmask MaskB = TRI->getSubRegIndexLaneMask(B); |
| unsigned NA = MaskA.getNumLanes(); |
| unsigned NB = MaskB.getNumLanes(); |
| if (NA != NB) |
| return NA > NB; |
| return MaskA.getHighestLane() > MaskB.getHighestLane(); |
| }); |
| |
| for (unsigned Idx : CoveringSubregs) { |
| LaneBitmask SubRegMask = TRI->getSubRegIndexLaneMask(Idx); |
| if ((SubRegMask & ~LaneMask).any() || (SubRegMask & LaneMask).none()) |
| continue; |
| |
| Func(Idx); |
| LaneMask &= ~SubRegMask; |
| if (LaneMask.none()) |
| return; |
| } |
| |
| llvm_unreachable("Failed to find all subregs to cover lane mask"); |
| } |
| |
| // Returns false if there is a use of a def already in the map. |
| // In this case we must break the clause. |
| bool SIFormMemoryClauses::canBundle(const MachineInstr &MI, |
| RegUse &Defs, RegUse &Uses) const { |
| // Check interference with defs. |
| for (const MachineOperand &MO : MI.operands()) { |
| // TODO: Prologue/Epilogue Insertion pass does not process bundled |
| // instructions. |
| if (MO.isFI()) |
| return false; |
| |
| if (!MO.isReg()) |
| continue; |
| |
| Register Reg = MO.getReg(); |
| |
| // If it is tied we will need to write same register as we read. |
| if (MO.isTied()) |
| return false; |
| |
| RegUse &Map = MO.isDef() ? Uses : Defs; |
| auto Conflict = Map.find(Reg); |
| if (Conflict == Map.end()) |
| continue; |
| |
| if (Register::isPhysicalRegister(Reg)) |
| return false; |
| |
| LaneBitmask Mask = TRI->getSubRegIndexLaneMask(MO.getSubReg()); |
| if ((Conflict->second.second & Mask).any()) |
| return false; |
| } |
| |
| return true; |
| } |
| |
| // Since all defs in the clause are early clobber we can run out of registers. |
| // Function returns false if pressure would hit the limit if instruction is |
| // bundled into a memory clause. |
| bool SIFormMemoryClauses::checkPressure(const MachineInstr &MI, |
| GCNDownwardRPTracker &RPT) { |
| // NB: skip advanceBeforeNext() call. Since all defs will be marked |
| // early-clobber they will all stay alive at least to the end of the |
| // clause. Therefor we should not decrease pressure even if load |
| // pointer becomes dead and could otherwise be reused for destination. |
| RPT.advanceToNext(); |
| GCNRegPressure MaxPressure = RPT.moveMaxPressure(); |
| unsigned Occupancy = MaxPressure.getOccupancy(*ST); |
| if (Occupancy >= MFI->getMinAllowedOccupancy() && |
| MaxPressure.getVGPRNum() <= MaxVGPRs && |
| MaxPressure.getSGPRNum() <= MaxSGPRs) { |
| LastRecordedOccupancy = Occupancy; |
| return true; |
| } |
| return false; |
| } |
| |
| // Collect register defs and uses along with their lane masks and states. |
| void SIFormMemoryClauses::collectRegUses(const MachineInstr &MI, |
| RegUse &Defs, RegUse &Uses) const { |
| for (const MachineOperand &MO : MI.operands()) { |
| if (!MO.isReg()) |
| continue; |
| Register Reg = MO.getReg(); |
| if (!Reg) |
| continue; |
| |
| LaneBitmask Mask = Register::isVirtualRegister(Reg) |
| ? TRI->getSubRegIndexLaneMask(MO.getSubReg()) |
| : LaneBitmask::getAll(); |
| RegUse &Map = MO.isDef() ? Defs : Uses; |
| |
| auto Loc = Map.find(Reg); |
| unsigned State = getMopState(MO); |
| if (Loc == Map.end()) { |
| Map[Reg] = std::make_pair(State, Mask); |
| } else { |
| Loc->second.first |= State; |
| Loc->second.second |= Mask; |
| } |
| } |
| } |
| |
| // Check register def/use conflicts, occupancy limits and collect def/use maps. |
| // Return true if instruction can be bundled with previous. It it cannot |
| // def/use maps are not updated. |
| bool SIFormMemoryClauses::processRegUses(const MachineInstr &MI, |
| RegUse &Defs, RegUse &Uses, |
| GCNDownwardRPTracker &RPT) { |
| if (!canBundle(MI, Defs, Uses)) |
| return false; |
| |
| if (!checkPressure(MI, RPT)) |
| return false; |
| |
| collectRegUses(MI, Defs, Uses); |
| return true; |
| } |
| |
| bool SIFormMemoryClauses::runOnMachineFunction(MachineFunction &MF) { |
| if (skipFunction(MF.getFunction())) |
| return false; |
| |
| ST = &MF.getSubtarget<GCNSubtarget>(); |
| if (!ST->isXNACKEnabled()) |
| return false; |
| |
| const SIInstrInfo *TII = ST->getInstrInfo(); |
| TRI = ST->getRegisterInfo(); |
| MRI = &MF.getRegInfo(); |
| MFI = MF.getInfo<SIMachineFunctionInfo>(); |
| LiveIntervals *LIS = &getAnalysis<LiveIntervals>(); |
| SlotIndexes *Ind = LIS->getSlotIndexes(); |
| bool Changed = false; |
| |
| MaxVGPRs = TRI->getAllocatableSet(MF, &AMDGPU::VGPR_32RegClass).count(); |
| MaxSGPRs = TRI->getAllocatableSet(MF, &AMDGPU::SGPR_32RegClass).count(); |
| unsigned FuncMaxClause = AMDGPU::getIntegerAttribute( |
| MF.getFunction(), "amdgpu-max-memory-clause", MaxClause); |
| |
| for (MachineBasicBlock &MBB : MF) { |
| MachineBasicBlock::instr_iterator Next; |
| for (auto I = MBB.instr_begin(), E = MBB.instr_end(); I != E; I = Next) { |
| MachineInstr &MI = *I; |
| Next = std::next(I); |
| |
| bool IsVMEM = isVMEMClauseInst(MI); |
| |
| if (!isValidClauseInst(MI, IsVMEM)) |
| continue; |
| |
| RegUse Defs, Uses; |
| GCNDownwardRPTracker RPT(*LIS); |
| RPT.reset(MI); |
| |
| if (!processRegUses(MI, Defs, Uses, RPT)) |
| continue; |
| |
| unsigned Length = 1; |
| for ( ; Next != E && Length < FuncMaxClause; ++Next) { |
| if (!isValidClauseInst(*Next, IsVMEM)) |
| break; |
| |
| // A load from pointer which was loaded inside the same bundle is an |
| // impossible clause because we will need to write and read the same |
| // register inside. In this case processRegUses will return false. |
| if (!processRegUses(*Next, Defs, Uses, RPT)) |
| break; |
| |
| ++Length; |
| } |
| if (Length < 2) |
| continue; |
| |
| Changed = true; |
| MFI->limitOccupancy(LastRecordedOccupancy); |
| |
| auto B = BuildMI(MBB, I, DebugLoc(), TII->get(TargetOpcode::BUNDLE)); |
| Ind->insertMachineInstrInMaps(*B); |
| |
| for (auto BI = I; BI != Next; ++BI) { |
| BI->bundleWithPred(); |
| Ind->removeSingleMachineInstrFromMaps(*BI); |
| |
| for (MachineOperand &MO : BI->defs()) |
| if (MO.readsReg()) |
| MO.setIsInternalRead(true); |
| } |
| |
| for (auto &&R : Defs) { |
| forAllLanes(R.first, R.second.second, [&R, &B](unsigned SubReg) { |
| unsigned S = R.second.first | RegState::EarlyClobber; |
| if (!SubReg) |
| S &= ~(RegState::Undef | RegState::Dead); |
| B.addDef(R.first, S, SubReg); |
| }); |
| } |
| |
| for (auto &&R : Uses) { |
| forAllLanes(R.first, R.second.second, [&R, &B](unsigned SubReg) { |
| B.addUse(R.first, R.second.first & ~RegState::Kill, SubReg); |
| }); |
| } |
| |
| for (auto &&R : Defs) { |
| unsigned Reg = R.first; |
| Uses.erase(Reg); |
| if (Register::isPhysicalRegister(Reg)) |
| continue; |
| LIS->removeInterval(Reg); |
| LIS->createAndComputeVirtRegInterval(Reg); |
| } |
| |
| for (auto &&R : Uses) { |
| unsigned Reg = R.first; |
| if (Register::isPhysicalRegister(Reg)) |
| continue; |
| LIS->removeInterval(Reg); |
| LIS->createAndComputeVirtRegInterval(Reg); |
| } |
| } |
| } |
| |
| return Changed; |
| } |