third_party/llvm-16.0/llvm/lib/Target/AMDGPU/AMDGPUReleaseVGPRs.cpp

//===- AMDGPUReleaseVGPRs.cpp - Automatically release vgprs on GFX11+ -----===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
/// \file
/// Insert S_SENDMSG instructions to release vgprs on GFX11+.
//
//===----------------------------------------------------------------------===//

#include "AMDGPU.h"
#include "AMDGPUSubtarget.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIDefines.h"
#include "llvm/ADT/DepthFirstIterator.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineOperand.h"
#include <optional>
using namespace llvm;

#define DEBUG_TYPE "release-vgprs"

namespace {

class AMDGPUReleaseVGPRs : public MachineFunctionPass {
public:
  static char ID;

  AMDGPUReleaseVGPRs() : MachineFunctionPass(ID) {}

  void getAnalysisUsage(AnalysisUsage &AU) const override {
    AU.setPreservesAll();
    MachineFunctionPass::getAnalysisUsage(AU);
  }

  // Track if the last instruction referencing a vgpr in a MBB is a VMEM
  // store. Because this pass is late in the pipeline, it is expected that the
  // last vgpr use will likely be one of vmem store, ds, exp.
  // Loads and others vgpr operations would have been
  // deleted by this point, except for complex control flow involving loops.
  // This is why we are just testing the type of instructions rather
  // than the operands.
  class LastVGPRUseIsVMEMStore {
    BitVector BlockVMEMStore;

    static std::optional<bool>
    lastVGPRUseIsStore(const MachineBasicBlock &MBB) {
      for (auto &MI : reverse(MBB.instrs())) {
        // If it's a VMEM store, a VGPR will be used, return true.
        if ((SIInstrInfo::isVMEM(MI) || SIInstrInfo::isFLAT(MI)) &&
            MI.mayStore())
          return true;

        // If it's referencing a VGPR but is not a VMEM store, return false.
        if (SIInstrInfo::isDS(MI) || SIInstrInfo::isEXP(MI) ||
            SIInstrInfo::isVMEM(MI) || SIInstrInfo::isFLAT(MI) ||
            SIInstrInfo::isVALU(MI))
          return false;
      }
      // Wait until the values are propagated from the predecessors
      return std::nullopt;
    }

  public:
    LastVGPRUseIsVMEMStore(const MachineFunction &MF)
        : BlockVMEMStore(MF.getNumBlockIDs()) {

      df_iterator_default_set<const MachineBasicBlock *> Visited;
      SmallVector<const MachineBasicBlock *> EndWithVMEMStoreBlocks;

      for (const auto &MBB : MF) {
        auto LastUseIsStore = lastVGPRUseIsStore(MBB);
        if (!LastUseIsStore.has_value())
          continue;

        if (*LastUseIsStore) {
          EndWithVMEMStoreBlocks.push_back(&MBB);
        } else {
          Visited.insert(&MBB);
        }
      }

      for (const auto *MBB : EndWithVMEMStoreBlocks) {
        for (const auto *Succ : depth_first_ext(MBB, Visited)) {
          BlockVMEMStore[Succ->getNumber()] = true;
        }
      }
    }

    // Return true if the last instruction referencing a vgpr in this MBB
    // is a VMEM store, otherwise return false.
    bool isLastVGPRUseVMEMStore(const MachineBasicBlock &MBB) const {
      return BlockVMEMStore[MBB.getNumber()];
    }
  };

  static bool
  runOnMachineBasicBlock(MachineBasicBlock &MBB, const SIInstrInfo *SII,
                         const LastVGPRUseIsVMEMStore &BlockVMEMStore) {

    bool Changed = false;

    for (auto &MI : MBB.terminators()) {
      // Look for S_ENDPGM instructions
      if (MI.getOpcode() == AMDGPU::S_ENDPGM ||
          MI.getOpcode() == AMDGPU::S_ENDPGM_SAVED) {
        // If the last instruction using a VGPR in the block is a VMEM store,
        // release VGPRs. The VGPRs release will be placed just before ending
        // the program
        if (BlockVMEMStore.isLastVGPRUseVMEMStore(MBB)) {
          BuildMI(MBB, MI, DebugLoc(), SII->get(AMDGPU::S_SENDMSG))
              .addImm(AMDGPU::SendMsg::ID_DEALLOC_VGPRS_GFX11Plus);
          Changed = true;
        }
      }
    }

    return Changed;
  }

  bool runOnMachineFunction(MachineFunction &MF) override {
    Function &F = MF.getFunction();
    if (skipFunction(F) || !AMDGPU::isEntryFunctionCC(F.getCallingConv()))
      return false;

    // This pass only runs on GFX11+
    const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
    if (ST.getGeneration() < AMDGPUSubtarget::GFX11)
      return false;

    LLVM_DEBUG(dbgs() << "AMDGPUReleaseVGPRs running on " << MF.getName()
                      << "\n");

    const SIInstrInfo *SII = ST.getInstrInfo();
    LastVGPRUseIsVMEMStore BlockVMEMStore(MF);

    bool Changed = false;
    for (auto &MBB : MF) {
      Changed |= runOnMachineBasicBlock(MBB, SII, BlockVMEMStore);
    }

    return Changed;
  }
};

} // namespace

char AMDGPUReleaseVGPRs::ID = 0;

char &llvm::AMDGPUReleaseVGPRsID = AMDGPUReleaseVGPRs::ID;

INITIALIZE_PASS(AMDGPUReleaseVGPRs, DEBUG_TYPE, "Release VGPRs", false, false)