third_party/llvm-16.0/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp - SwiftShader - Git at Google


            
            //===-- RISCVExpandPseudoInsts.cpp - Expand pseudo instructions -----------===//

            
            //

            
            // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

            
            // See https://llvm.org/LICENSE.txt for license information.

            
            // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

            
            //

            
            //===----------------------------------------------------------------------===//

            
            //

            
            // This file contains a pass that expands pseudo instructions into target

            
            // instructions. This pass should be run after register allocation but before

            
            // the post-regalloc scheduling pass.

            
            //

            
            //===----------------------------------------------------------------------===//


            #include "RISCV.h"

            
            #include "RISCVInstrInfo.h"

            
            #include "RISCVTargetMachine.h"


            #include "llvm/CodeGen/LivePhysRegs.h"

            
            #include "llvm/CodeGen/MachineFunctionPass.h"

            
            #include "llvm/CodeGen/MachineInstrBuilder.h"

            
            #include "llvm/MC/MCContext.h"


            using namespace llvm;


            #define RISCV_EXPAND_PSEUDO_NAME "RISCV pseudo instruction expansion pass"

            
            #define RISCV_PRERA_EXPAND_PSEUDO_NAME "RISCV Pre-RA pseudo instruction expansion pass"


            namespace {


            class RISCVExpandPseudo : public MachineFunctionPass {

            
            public:

            
              const RISCVInstrInfo *TII;

            
              static char ID;


              RISCVExpandPseudo() : MachineFunctionPass(ID) {

            
                initializeRISCVExpandPseudoPass(*PassRegistry::getPassRegistry());

            
              }


              bool runOnMachineFunction(MachineFunction &MF) override;


              StringRef getPassName() const override { return RISCV_EXPAND_PSEUDO_NAME; }


            private:

            
              bool expandMBB(MachineBasicBlock &MBB);

            
              bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,

            
                            MachineBasicBlock::iterator &NextMBBI);

            
              bool expandCCOp(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,

            
                              MachineBasicBlock::iterator &NextMBBI);

            
              bool expandVSetVL(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI);

            
              bool expandVMSET_VMCLR(MachineBasicBlock &MBB,

            
                                     MachineBasicBlock::iterator MBBI, unsigned Opcode);

            
            };


            char RISCVExpandPseudo::ID = 0;


            bool RISCVExpandPseudo::runOnMachineFunction(MachineFunction &MF) {

            
              TII = static_cast<const RISCVInstrInfo *>(MF.getSubtarget().getInstrInfo());

            
              bool Modified = false;

            
              for (auto &MBB : MF)

            
                Modified |= expandMBB(MBB);

            
              return Modified;

            
            }


            bool RISCVExpandPseudo::expandMBB(MachineBasicBlock &MBB) {

            
              bool Modified = false;


              MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();

            
              while (MBBI != E) {

            
                MachineBasicBlock::iterator NMBBI = std::next(MBBI);

            
                Modified |= expandMI(MBB, MBBI, NMBBI);

            
                MBBI = NMBBI;

            
              }


              return Modified;

            
            }


            bool RISCVExpandPseudo::expandMI(MachineBasicBlock &MBB,

            
                                             MachineBasicBlock::iterator MBBI,

            
                                             MachineBasicBlock::iterator &NextMBBI) {

            
              // RISCVInstrInfo::getInstSizeInBytes expects that the total size of the

            
              // expanded instructions for each pseudo is correct in the Size field of the

            
              // tablegen definition for the pseudo.

            
              switch (MBBI->getOpcode()) {

            
              case RISCV::PseudoCCMOVGPR:

            
              case RISCV::PseudoCCADD:

            
              case RISCV::PseudoCCSUB:

            
              case RISCV::PseudoCCAND:

            
              case RISCV::PseudoCCOR:

            
              case RISCV::PseudoCCXOR:

            
              case RISCV::PseudoCCADDW:

            
              case RISCV::PseudoCCSUBW:

            
                return expandCCOp(MBB, MBBI, NextMBBI);

            
              case RISCV::PseudoVSETVLI:

            
              case RISCV::PseudoVSETVLIX0:

            
              case RISCV::PseudoVSETIVLI:

            
                return expandVSetVL(MBB, MBBI);

            
              case RISCV::PseudoVMCLR_M_B1:

            
              case RISCV::PseudoVMCLR_M_B2:

            
              case RISCV::PseudoVMCLR_M_B4:

            
              case RISCV::PseudoVMCLR_M_B8:

            
              case RISCV::PseudoVMCLR_M_B16:

            
              case RISCV::PseudoVMCLR_M_B32:

            
              case RISCV::PseudoVMCLR_M_B64:

            
                // vmclr.m vd => vmxor.mm vd, vd, vd

            
                return expandVMSET_VMCLR(MBB, MBBI, RISCV::VMXOR_MM);

            
              case RISCV::PseudoVMSET_M_B1:

            
              case RISCV::PseudoVMSET_M_B2:

            
              case RISCV::PseudoVMSET_M_B4:

            
              case RISCV::PseudoVMSET_M_B8:

            
              case RISCV::PseudoVMSET_M_B16:

            
              case RISCV::PseudoVMSET_M_B32:

            
              case RISCV::PseudoVMSET_M_B64:

            
                // vmset.m vd => vmxnor.mm vd, vd, vd

            
                return expandVMSET_VMCLR(MBB, MBBI, RISCV::VMXNOR_MM);

            
              }


              return false;

            
            }


            bool RISCVExpandPseudo::expandCCOp(MachineBasicBlock &MBB,

            
                                               MachineBasicBlock::iterator MBBI,

            
                                               MachineBasicBlock::iterator &NextMBBI) {


              MachineFunction *MF = MBB.getParent();

            
              MachineInstr &MI = *MBBI;

            
              DebugLoc DL = MI.getDebugLoc();


              MachineBasicBlock *TrueBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());

            
              MachineBasicBlock *MergeBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());


              MF->insert(++MBB.getIterator(), TrueBB);

            
              MF->insert(++TrueBB->getIterator(), MergeBB);


              // We want to copy the "true" value when the condition is true which means

            
              // we need to invert the branch condition to jump over TrueBB when the

            
              // condition is false.

            
              auto CC = static_cast<RISCVCC::CondCode>(MI.getOperand(3).getImm());

            
              CC = RISCVCC::getOppositeBranchCondition(CC);


              // Insert branch instruction.

            
              BuildMI(MBB, MBBI, DL, TII->getBrCond(CC))

            
                  .addReg(MI.getOperand(1).getReg())

            
                  .addReg(MI.getOperand(2).getReg())

            
                  .addMBB(MergeBB);


              Register DestReg = MI.getOperand(0).getReg();

            
              assert(MI.getOperand(4).getReg() == DestReg);


              if (MI.getOpcode() == RISCV::PseudoCCMOVGPR) {

            
                // Add MV.

            
                BuildMI(TrueBB, DL, TII->get(RISCV::ADDI), DestReg)

            
                    .add(MI.getOperand(5))

            
                    .addImm(0);

            
              } else {

            
                unsigned NewOpc;

            
                switch (MI.getOpcode()) {

            
                default:

            
                  llvm_unreachable("Unexpected opcode!");

            
                case RISCV::PseudoCCADD:   NewOpc = RISCV::ADD;   break;

            
                case RISCV::PseudoCCSUB:   NewOpc = RISCV::SUB;   break;

            
                case RISCV::PseudoCCAND:   NewOpc = RISCV::AND;   break;

            
                case RISCV::PseudoCCOR:    NewOpc = RISCV::OR;    break;

            
                case RISCV::PseudoCCXOR:   NewOpc = RISCV::XOR;   break;

            
                case RISCV::PseudoCCADDW:  NewOpc = RISCV::ADDW;  break;

            
                case RISCV::PseudoCCSUBW:  NewOpc = RISCV::SUBW;  break;

            
                }

            
                BuildMI(TrueBB, DL, TII->get(NewOpc), DestReg)

            
                    .add(MI.getOperand(5))

            
                    .add(MI.getOperand(6));

            
              }


              TrueBB->addSuccessor(MergeBB);


              MergeBB->splice(MergeBB->end(), &MBB, MI, MBB.end());

            
              MergeBB->transferSuccessors(&MBB);


              MBB.addSuccessor(TrueBB);

            
              MBB.addSuccessor(MergeBB);


              NextMBBI = MBB.end();

            
              MI.eraseFromParent();


              // Make sure live-ins are correctly attached to this new basic block.

            
              LivePhysRegs LiveRegs;

            
              computeAndAddLiveIns(LiveRegs, *TrueBB);

            
              computeAndAddLiveIns(LiveRegs, *MergeBB);


              return true;

            
            }


            bool RISCVExpandPseudo::expandVSetVL(MachineBasicBlock &MBB,

            
                                                 MachineBasicBlock::iterator MBBI) {

            
              assert(MBBI->getNumExplicitOperands() == 3 && MBBI->getNumOperands() >= 5 &&

            
                     "Unexpected instruction format");


              DebugLoc DL = MBBI->getDebugLoc();


              assert((MBBI->getOpcode() == RISCV::PseudoVSETVLI ||

            
                      MBBI->getOpcode() == RISCV::PseudoVSETVLIX0 ||

            
                      MBBI->getOpcode() == RISCV::PseudoVSETIVLI) &&

            
                     "Unexpected pseudo instruction");

            
              unsigned Opcode;

            
              if (MBBI->getOpcode() == RISCV::PseudoVSETIVLI)

            
                Opcode = RISCV::VSETIVLI;

            
              else

            
                Opcode = RISCV::VSETVLI;

            
              const MCInstrDesc &Desc = TII->get(Opcode);

            
              assert(Desc.getNumOperands() == 3 && "Unexpected instruction format");


              Register DstReg = MBBI->getOperand(0).getReg();

            
              bool DstIsDead = MBBI->getOperand(0).isDead();

            
              BuildMI(MBB, MBBI, DL, Desc)

            
                  .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))

            
                  .add(MBBI->getOperand(1))  // VL

            
                  .add(MBBI->getOperand(2)); // VType


              MBBI->eraseFromParent(); // The pseudo instruction is gone now.

            
              return true;

            
            }


            bool RISCVExpandPseudo::expandVMSET_VMCLR(MachineBasicBlock &MBB,

            
                                                      MachineBasicBlock::iterator MBBI,

            
                                                      unsigned Opcode) {

            
              DebugLoc DL = MBBI->getDebugLoc();

            
              Register DstReg = MBBI->getOperand(0).getReg();

            
              const MCInstrDesc &Desc = TII->get(Opcode);

            
              BuildMI(MBB, MBBI, DL, Desc, DstReg)

            
                  .addReg(DstReg, RegState::Undef)

            
                  .addReg(DstReg, RegState::Undef);

            
              MBBI->eraseFromParent(); // The pseudo instruction is gone now.

            
              return true;

            
            }


            class RISCVPreRAExpandPseudo : public MachineFunctionPass {

            
            public:

            
              const RISCVInstrInfo *TII;

            
              static char ID;


              RISCVPreRAExpandPseudo() : MachineFunctionPass(ID) {

            
                initializeRISCVPreRAExpandPseudoPass(*PassRegistry::getPassRegistry());

            
              }


              bool runOnMachineFunction(MachineFunction &MF) override;


              void getAnalysisUsage(AnalysisUsage &AU) const override {

            
                AU.setPreservesCFG();

            
                MachineFunctionPass::getAnalysisUsage(AU);

            
              }

            
              StringRef getPassName() const override {

            
                return RISCV_PRERA_EXPAND_PSEUDO_NAME;

            
              }


            private:

            
              bool expandMBB(MachineBasicBlock &MBB);

            
              bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,

            
                            MachineBasicBlock::iterator &NextMBBI);

            
              bool expandAuipcInstPair(MachineBasicBlock &MBB,

            
                                       MachineBasicBlock::iterator MBBI,

            
                                       MachineBasicBlock::iterator &NextMBBI,

            
                                       unsigned FlagsHi, unsigned SecondOpcode);

            
              bool expandLoadLocalAddress(MachineBasicBlock &MBB,

            
                                          MachineBasicBlock::iterator MBBI,

            
                                          MachineBasicBlock::iterator &NextMBBI);

            
              bool expandLoadAddress(MachineBasicBlock &MBB,

            
                                     MachineBasicBlock::iterator MBBI,

            
                                     MachineBasicBlock::iterator &NextMBBI);

            
              bool expandLoadTLSIEAddress(MachineBasicBlock &MBB,

            
                                          MachineBasicBlock::iterator MBBI,

            
                                          MachineBasicBlock::iterator &NextMBBI);

            
              bool expandLoadTLSGDAddress(MachineBasicBlock &MBB,

            
                                          MachineBasicBlock::iterator MBBI,

            
                                          MachineBasicBlock::iterator &NextMBBI);

            
            };


            char RISCVPreRAExpandPseudo::ID = 0;


            bool RISCVPreRAExpandPseudo::runOnMachineFunction(MachineFunction &MF) {

            
              TII = static_cast<const RISCVInstrInfo *>(MF.getSubtarget().getInstrInfo());

            
              bool Modified = false;

            
              for (auto &MBB : MF)

            
                Modified |= expandMBB(MBB);

            
              return Modified;

            
            }


            bool RISCVPreRAExpandPseudo::expandMBB(MachineBasicBlock &MBB) {

            
              bool Modified = false;


              MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();

            
              while (MBBI != E) {

            
                MachineBasicBlock::iterator NMBBI = std::next(MBBI);

            
                Modified |= expandMI(MBB, MBBI, NMBBI);

            
                MBBI = NMBBI;

            
              }


              return Modified;

            
            }


            bool RISCVPreRAExpandPseudo::expandMI(MachineBasicBlock &MBB,

            
                                                  MachineBasicBlock::iterator MBBI,

            
                                                  MachineBasicBlock::iterator &NextMBBI) {


              switch (MBBI->getOpcode()) {

            
              case RISCV::PseudoLLA:

            
                return expandLoadLocalAddress(MBB, MBBI, NextMBBI);

            
              case RISCV::PseudoLA:

            
                return expandLoadAddress(MBB, MBBI, NextMBBI);

            
              case RISCV::PseudoLA_TLS_IE:

            
                return expandLoadTLSIEAddress(MBB, MBBI, NextMBBI);

            
              case RISCV::PseudoLA_TLS_GD:

            
                return expandLoadTLSGDAddress(MBB, MBBI, NextMBBI);

            
              }

            
              return false;

            
            }


            bool RISCVPreRAExpandPseudo::expandAuipcInstPair(

            
                MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,

            
                MachineBasicBlock::iterator &NextMBBI, unsigned FlagsHi,

            
                unsigned SecondOpcode) {

            
              MachineFunction *MF = MBB.getParent();

            
              MachineInstr &MI = *MBBI;

            
              DebugLoc DL = MI.getDebugLoc();


              Register DestReg = MI.getOperand(0).getReg();

            
              Register ScratchReg =

            
                  MF->getRegInfo().createVirtualRegister(&RISCV::GPRRegClass);


              MachineOperand &Symbol = MI.getOperand(1);

            
              Symbol.setTargetFlags(FlagsHi);

            
              MCSymbol *AUIPCSymbol = MF->getContext().createNamedTempSymbol("pcrel_hi");


              MachineInstr *MIAUIPC =

            
                  BuildMI(MBB, MBBI, DL, TII->get(RISCV::AUIPC), ScratchReg).add(Symbol);

            
              MIAUIPC->setPreInstrSymbol(*MF, AUIPCSymbol);


              MachineInstr *SecondMI =

            
                  BuildMI(MBB, MBBI, DL, TII->get(SecondOpcode), DestReg)

            
                      .addReg(ScratchReg)

            
                      .addSym(AUIPCSymbol, RISCVII::MO_PCREL_LO);


              if (MI.hasOneMemOperand())

            
                SecondMI->addMemOperand(*MF, *MI.memoperands_begin());


              MI.eraseFromParent();

            
              return true;

            
            }


            bool RISCVPreRAExpandPseudo::expandLoadLocalAddress(

            
                MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,

            
                MachineBasicBlock::iterator &NextMBBI) {

            
              return expandAuipcInstPair(MBB, MBBI, NextMBBI, RISCVII::MO_PCREL_HI,

            
                                         RISCV::ADDI);

            
            }


            bool RISCVPreRAExpandPseudo::expandLoadAddress(

            
                MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,

            
                MachineBasicBlock::iterator &NextMBBI) {

            
              MachineFunction *MF = MBB.getParent();


              const auto &STI = MF->getSubtarget<RISCVSubtarget>();

            
              // When HWASAN is used and tagging of global variables is enabled

            
              // they should be accessed via the GOT, since the tagged address of a global

            
              // is incompatible with existing code models. This also applies to non-pic

            
              // mode.

            
              assert(MF->getTarget().isPositionIndependent() || STI.allowTaggedGlobals());

            
              unsigned SecondOpcode = STI.is64Bit() ? RISCV::LD : RISCV::LW;

            
              return expandAuipcInstPair(MBB, MBBI, NextMBBI, RISCVII::MO_GOT_HI,

            
                                         SecondOpcode);

            
            }


            bool RISCVPreRAExpandPseudo::expandLoadTLSIEAddress(

            
                MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,

            
                MachineBasicBlock::iterator &NextMBBI) {

            
              MachineFunction *MF = MBB.getParent();


              const auto &STI = MF->getSubtarget<RISCVSubtarget>();

            
              unsigned SecondOpcode = STI.is64Bit() ? RISCV::LD : RISCV::LW;

            
              return expandAuipcInstPair(MBB, MBBI, NextMBBI, RISCVII::MO_TLS_GOT_HI,

            
                                         SecondOpcode);

            
            }


            bool RISCVPreRAExpandPseudo::expandLoadTLSGDAddress(

            
                MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,

            
                MachineBasicBlock::iterator &NextMBBI) {

            
              return expandAuipcInstPair(MBB, MBBI, NextMBBI, RISCVII::MO_TLS_GD_HI,

            
                                         RISCV::ADDI);

            
            }


            } // end of anonymous namespace


            INITIALIZE_PASS(RISCVExpandPseudo, "riscv-expand-pseudo",

            
                            RISCV_EXPAND_PSEUDO_NAME, false, false)


            INITIALIZE_PASS(RISCVPreRAExpandPseudo, "riscv-prera-expand-pseudo",

            
                            RISCV_PRERA_EXPAND_PSEUDO_NAME, false, false)


            namespace llvm {


            FunctionPass *createRISCVExpandPseudoPass() { return new RISCVExpandPseudo(); }

            
            FunctionPass *createRISCVPreRAExpandPseudoPass() { return new RISCVPreRAExpandPseudo(); }


            } // end of namespace llvm
	//===-- RISCVExpandPseudoInsts.cpp - Expand pseudo instructions -----------===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	//
	// This file contains a pass that expands pseudo instructions into target
	// instructions. This pass should be run after register allocation but before
	// the post-regalloc scheduling pass.
	//
	//===----------------------------------------------------------------------===//

	#include "RISCV.h"
	#include "RISCVInstrInfo.h"
	#include "RISCVTargetMachine.h"

	#include "llvm/CodeGen/LivePhysRegs.h"
	#include "llvm/CodeGen/MachineFunctionPass.h"
	#include "llvm/CodeGen/MachineInstrBuilder.h"
	#include "llvm/MC/MCContext.h"

	using namespace llvm;

	#define RISCV_EXPAND_PSEUDO_NAME "RISCV pseudo instruction expansion pass"
	#define RISCV_PRERA_EXPAND_PSEUDO_NAME "RISCV Pre-RA pseudo instruction expansion pass"

	namespace {

	class RISCVExpandPseudo : public MachineFunctionPass {
	public:
	const RISCVInstrInfo *TII;
	static char ID;

	RISCVExpandPseudo() : MachineFunctionPass(ID) {
	initializeRISCVExpandPseudoPass(*PassRegistry::getPassRegistry());
	}

	bool runOnMachineFunction(MachineFunction &MF) override;

	StringRef getPassName() const override { return RISCV_EXPAND_PSEUDO_NAME; }

	private:
	bool expandMBB(MachineBasicBlock &MBB);
	bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
	MachineBasicBlock::iterator &NextMBBI);
	bool expandCCOp(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
	MachineBasicBlock::iterator &NextMBBI);
	bool expandVSetVL(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI);
	bool expandVMSET_VMCLR(MachineBasicBlock &MBB,
	MachineBasicBlock::iterator MBBI, unsigned Opcode);
	};

	char RISCVExpandPseudo::ID = 0;

	bool RISCVExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
	TII = static_cast<const RISCVInstrInfo *>(MF.getSubtarget().getInstrInfo());
	bool Modified = false;
	for (auto &MBB : MF)
	Modified \|= expandMBB(MBB);
	return Modified;
	}

	bool RISCVExpandPseudo::expandMBB(MachineBasicBlock &MBB) {
	bool Modified = false;

	MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
	while (MBBI != E) {
	MachineBasicBlock::iterator NMBBI = std::next(MBBI);
	Modified \|= expandMI(MBB, MBBI, NMBBI);
	MBBI = NMBBI;
	}

	return Modified;
	}

	bool RISCVExpandPseudo::expandMI(MachineBasicBlock &MBB,
	MachineBasicBlock::iterator MBBI,
	MachineBasicBlock::iterator &NextMBBI) {
	// RISCVInstrInfo::getInstSizeInBytes expects that the total size of the
	// expanded instructions for each pseudo is correct in the Size field of the
	// tablegen definition for the pseudo.
	switch (MBBI->getOpcode()) {
	case RISCV::PseudoCCMOVGPR:
	case RISCV::PseudoCCADD:
	case RISCV::PseudoCCSUB:
	case RISCV::PseudoCCAND:
	case RISCV::PseudoCCOR:
	case RISCV::PseudoCCXOR:
	case RISCV::PseudoCCADDW:
	case RISCV::PseudoCCSUBW:
	return expandCCOp(MBB, MBBI, NextMBBI);
	case RISCV::PseudoVSETVLI:
	case RISCV::PseudoVSETVLIX0:
	case RISCV::PseudoVSETIVLI:
	return expandVSetVL(MBB, MBBI);
	case RISCV::PseudoVMCLR_M_B1:
	case RISCV::PseudoVMCLR_M_B2:
	case RISCV::PseudoVMCLR_M_B4:
	case RISCV::PseudoVMCLR_M_B8:
	case RISCV::PseudoVMCLR_M_B16:
	case RISCV::PseudoVMCLR_M_B32:
	case RISCV::PseudoVMCLR_M_B64:
	// vmclr.m vd => vmxor.mm vd, vd, vd
	return expandVMSET_VMCLR(MBB, MBBI, RISCV::VMXOR_MM);
	case RISCV::PseudoVMSET_M_B1:
	case RISCV::PseudoVMSET_M_B2:
	case RISCV::PseudoVMSET_M_B4:
	case RISCV::PseudoVMSET_M_B8:
	case RISCV::PseudoVMSET_M_B16:
	case RISCV::PseudoVMSET_M_B32:
	case RISCV::PseudoVMSET_M_B64:
	// vmset.m vd => vmxnor.mm vd, vd, vd
	return expandVMSET_VMCLR(MBB, MBBI, RISCV::VMXNOR_MM);
	}

	return false;
	}

	bool RISCVExpandPseudo::expandCCOp(MachineBasicBlock &MBB,
	MachineBasicBlock::iterator MBBI,
	MachineBasicBlock::iterator &NextMBBI) {

	MachineFunction *MF = MBB.getParent();
	MachineInstr &MI = *MBBI;
	DebugLoc DL = MI.getDebugLoc();

	MachineBasicBlock *TrueBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
	MachineBasicBlock *MergeBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());

	MF->insert(++MBB.getIterator(), TrueBB);
	MF->insert(++TrueBB->getIterator(), MergeBB);

	// We want to copy the "true" value when the condition is true which means
	// we need to invert the branch condition to jump over TrueBB when the
	// condition is false.
	auto CC = static_cast<RISCVCC::CondCode>(MI.getOperand(3).getImm());
	CC = RISCVCC::getOppositeBranchCondition(CC);

	// Insert branch instruction.
	BuildMI(MBB, MBBI, DL, TII->getBrCond(CC))
	.addReg(MI.getOperand(1).getReg())
	.addReg(MI.getOperand(2).getReg())
	.addMBB(MergeBB);

	Register DestReg = MI.getOperand(0).getReg();
	assert(MI.getOperand(4).getReg() == DestReg);

	if (MI.getOpcode() == RISCV::PseudoCCMOVGPR) {
	// Add MV.
	BuildMI(TrueBB, DL, TII->get(RISCV::ADDI), DestReg)
	.add(MI.getOperand(5))
	.addImm(0);
	} else {
	unsigned NewOpc;
	switch (MI.getOpcode()) {
	default:
	llvm_unreachable("Unexpected opcode!");
	case RISCV::PseudoCCADD: NewOpc = RISCV::ADD; break;
	case RISCV::PseudoCCSUB: NewOpc = RISCV::SUB; break;
	case RISCV::PseudoCCAND: NewOpc = RISCV::AND; break;
	case RISCV::PseudoCCOR: NewOpc = RISCV::OR; break;
	case RISCV::PseudoCCXOR: NewOpc = RISCV::XOR; break;
	case RISCV::PseudoCCADDW: NewOpc = RISCV::ADDW; break;
	case RISCV::PseudoCCSUBW: NewOpc = RISCV::SUBW; break;
	}
	BuildMI(TrueBB, DL, TII->get(NewOpc), DestReg)
	.add(MI.getOperand(5))
	.add(MI.getOperand(6));
	}

	TrueBB->addSuccessor(MergeBB);

	MergeBB->splice(MergeBB->end(), &MBB, MI, MBB.end());
	MergeBB->transferSuccessors(&MBB);

	MBB.addSuccessor(TrueBB);
	MBB.addSuccessor(MergeBB);

	NextMBBI = MBB.end();
	MI.eraseFromParent();

	// Make sure live-ins are correctly attached to this new basic block.
	LivePhysRegs LiveRegs;
	computeAndAddLiveIns(LiveRegs, *TrueBB);
	computeAndAddLiveIns(LiveRegs, *MergeBB);

	return true;
	}

	bool RISCVExpandPseudo::expandVSetVL(MachineBasicBlock &MBB,
	MachineBasicBlock::iterator MBBI) {
	assert(MBBI->getNumExplicitOperands() == 3 && MBBI->getNumOperands() >= 5 &&
	"Unexpected instruction format");

	DebugLoc DL = MBBI->getDebugLoc();

	assert((MBBI->getOpcode() == RISCV::PseudoVSETVLI \|\|
	MBBI->getOpcode() == RISCV::PseudoVSETVLIX0 \|\|
	MBBI->getOpcode() == RISCV::PseudoVSETIVLI) &&
	"Unexpected pseudo instruction");
	unsigned Opcode;
	if (MBBI->getOpcode() == RISCV::PseudoVSETIVLI)
	Opcode = RISCV::VSETIVLI;
	else
	Opcode = RISCV::VSETVLI;
	const MCInstrDesc &Desc = TII->get(Opcode);
	assert(Desc.getNumOperands() == 3 && "Unexpected instruction format");

	Register DstReg = MBBI->getOperand(0).getReg();
	bool DstIsDead = MBBI->getOperand(0).isDead();
	BuildMI(MBB, MBBI, DL, Desc)
	.addReg(DstReg, RegState::Define \| getDeadRegState(DstIsDead))
	.add(MBBI->getOperand(1)) // VL
	.add(MBBI->getOperand(2)); // VType

	MBBI->eraseFromParent(); // The pseudo instruction is gone now.
	return true;
	}

	bool RISCVExpandPseudo::expandVMSET_VMCLR(MachineBasicBlock &MBB,
	MachineBasicBlock::iterator MBBI,
	unsigned Opcode) {
	DebugLoc DL = MBBI->getDebugLoc();
	Register DstReg = MBBI->getOperand(0).getReg();
	const MCInstrDesc &Desc = TII->get(Opcode);
	BuildMI(MBB, MBBI, DL, Desc, DstReg)
	.addReg(DstReg, RegState::Undef)
	.addReg(DstReg, RegState::Undef);
	MBBI->eraseFromParent(); // The pseudo instruction is gone now.
	return true;
	}

	class RISCVPreRAExpandPseudo : public MachineFunctionPass {
	public:
	const RISCVInstrInfo *TII;
	static char ID;

	RISCVPreRAExpandPseudo() : MachineFunctionPass(ID) {
	initializeRISCVPreRAExpandPseudoPass(*PassRegistry::getPassRegistry());
	}

	bool runOnMachineFunction(MachineFunction &MF) override;

	void getAnalysisUsage(AnalysisUsage &AU) const override {
	AU.setPreservesCFG();
	MachineFunctionPass::getAnalysisUsage(AU);
	}
	StringRef getPassName() const override {
	return RISCV_PRERA_EXPAND_PSEUDO_NAME;
	}

	private:
	bool expandMBB(MachineBasicBlock &MBB);
	bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
	MachineBasicBlock::iterator &NextMBBI);
	bool expandAuipcInstPair(MachineBasicBlock &MBB,
	MachineBasicBlock::iterator MBBI,
	MachineBasicBlock::iterator &NextMBBI,
	unsigned FlagsHi, unsigned SecondOpcode);
	bool expandLoadLocalAddress(MachineBasicBlock &MBB,
	MachineBasicBlock::iterator MBBI,
	MachineBasicBlock::iterator &NextMBBI);
	bool expandLoadAddress(MachineBasicBlock &MBB,
	MachineBasicBlock::iterator MBBI,
	MachineBasicBlock::iterator &NextMBBI);
	bool expandLoadTLSIEAddress(MachineBasicBlock &MBB,
	MachineBasicBlock::iterator MBBI,
	MachineBasicBlock::iterator &NextMBBI);
	bool expandLoadTLSGDAddress(MachineBasicBlock &MBB,
	MachineBasicBlock::iterator MBBI,
	MachineBasicBlock::iterator &NextMBBI);
	};

	char RISCVPreRAExpandPseudo::ID = 0;

	bool RISCVPreRAExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
	TII = static_cast<const RISCVInstrInfo *>(MF.getSubtarget().getInstrInfo());
	bool Modified = false;
	for (auto &MBB : MF)
	Modified \|= expandMBB(MBB);
	return Modified;
	}

	bool RISCVPreRAExpandPseudo::expandMBB(MachineBasicBlock &MBB) {
	bool Modified = false;

	MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
	while (MBBI != E) {
	MachineBasicBlock::iterator NMBBI = std::next(MBBI);
	Modified \|= expandMI(MBB, MBBI, NMBBI);
	MBBI = NMBBI;
	}

	return Modified;
	}

	bool RISCVPreRAExpandPseudo::expandMI(MachineBasicBlock &MBB,
	MachineBasicBlock::iterator MBBI,
	MachineBasicBlock::iterator &NextMBBI) {

	switch (MBBI->getOpcode()) {
	case RISCV::PseudoLLA:
	return expandLoadLocalAddress(MBB, MBBI, NextMBBI);
	case RISCV::PseudoLA:
	return expandLoadAddress(MBB, MBBI, NextMBBI);
	case RISCV::PseudoLA_TLS_IE:
	return expandLoadTLSIEAddress(MBB, MBBI, NextMBBI);
	case RISCV::PseudoLA_TLS_GD:
	return expandLoadTLSGDAddress(MBB, MBBI, NextMBBI);
	}
	return false;
	}

	bool RISCVPreRAExpandPseudo::expandAuipcInstPair(
	MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
	MachineBasicBlock::iterator &NextMBBI, unsigned FlagsHi,
	unsigned SecondOpcode) {
	MachineFunction *MF = MBB.getParent();
	MachineInstr &MI = *MBBI;
	DebugLoc DL = MI.getDebugLoc();

	Register DestReg = MI.getOperand(0).getReg();
	Register ScratchReg =
	MF->getRegInfo().createVirtualRegister(&RISCV::GPRRegClass);

	MachineOperand &Symbol = MI.getOperand(1);
	Symbol.setTargetFlags(FlagsHi);
	MCSymbol *AUIPCSymbol = MF->getContext().createNamedTempSymbol("pcrel_hi");

	MachineInstr *MIAUIPC =
	BuildMI(MBB, MBBI, DL, TII->get(RISCV::AUIPC), ScratchReg).add(Symbol);
	MIAUIPC->setPreInstrSymbol(*MF, AUIPCSymbol);

	MachineInstr *SecondMI =
	BuildMI(MBB, MBBI, DL, TII->get(SecondOpcode), DestReg)
	.addReg(ScratchReg)
	.addSym(AUIPCSymbol, RISCVII::MO_PCREL_LO);

	if (MI.hasOneMemOperand())
	SecondMI->addMemOperand(MF, MI.memoperands_begin());

	MI.eraseFromParent();
	return true;
	}

	bool RISCVPreRAExpandPseudo::expandLoadLocalAddress(
	MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
	MachineBasicBlock::iterator &NextMBBI) {
	return expandAuipcInstPair(MBB, MBBI, NextMBBI, RISCVII::MO_PCREL_HI,
	RISCV::ADDI);
	}

	bool RISCVPreRAExpandPseudo::expandLoadAddress(
	MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
	MachineBasicBlock::iterator &NextMBBI) {
	MachineFunction *MF = MBB.getParent();

	const auto &STI = MF->getSubtarget<RISCVSubtarget>();
	// When HWASAN is used and tagging of global variables is enabled
	// they should be accessed via the GOT, since the tagged address of a global
	// is incompatible with existing code models. This also applies to non-pic
	// mode.
	assert(MF->getTarget().isPositionIndependent() \|\| STI.allowTaggedGlobals());
	unsigned SecondOpcode = STI.is64Bit() ? RISCV::LD : RISCV::LW;
	return expandAuipcInstPair(MBB, MBBI, NextMBBI, RISCVII::MO_GOT_HI,
	SecondOpcode);
	}

	bool RISCVPreRAExpandPseudo::expandLoadTLSIEAddress(
	MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
	MachineBasicBlock::iterator &NextMBBI) {
	MachineFunction *MF = MBB.getParent();

	const auto &STI = MF->getSubtarget<RISCVSubtarget>();
	unsigned SecondOpcode = STI.is64Bit() ? RISCV::LD : RISCV::LW;
	return expandAuipcInstPair(MBB, MBBI, NextMBBI, RISCVII::MO_TLS_GOT_HI,
	SecondOpcode);
	}

	bool RISCVPreRAExpandPseudo::expandLoadTLSGDAddress(
	MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
	MachineBasicBlock::iterator &NextMBBI) {
	return expandAuipcInstPair(MBB, MBBI, NextMBBI, RISCVII::MO_TLS_GD_HI,
	RISCV::ADDI);
	}

	} // end of anonymous namespace

	INITIALIZE_PASS(RISCVExpandPseudo, "riscv-expand-pseudo",
	RISCV_EXPAND_PSEUDO_NAME, false, false)

	INITIALIZE_PASS(RISCVPreRAExpandPseudo, "riscv-prera-expand-pseudo",
	RISCV_PRERA_EXPAND_PSEUDO_NAME, false, false)

	namespace llvm {

	FunctionPass *createRISCVExpandPseudoPass() { return new RISCVExpandPseudo(); }
	FunctionPass *createRISCVPreRAExpandPseudoPass() { return new RISCVPreRAExpandPseudo(); }

	} // end of namespace llvm