| //===- AArch64ExpandPseudoInsts.cpp - Expand pseudo instructions ----------===// | |
| // | |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |
| // See https://llvm.org/LICENSE.txt for license information. | |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| // | |
| //===----------------------------------------------------------------------===// | |
| // | |
| // This file contains a pass that expands pseudo instructions into target | |
| // instructions to allow proper scheduling and other late optimizations. This | |
| // pass should be run after register allocation but before the post-regalloc | |
| // scheduling pass. | |
| // | |
| //===----------------------------------------------------------------------===// | |
| #include "AArch64ExpandImm.h" | |
| #include "AArch64InstrInfo.h" | |
| #include "AArch64MachineFunctionInfo.h" | |
| #include "AArch64Subtarget.h" | |
| #include "MCTargetDesc/AArch64AddressingModes.h" | |
| #include "Utils/AArch64BaseInfo.h" | |
| #include "llvm/ADT/DenseMap.h" | |
| #include "llvm/ADT/Triple.h" | |
| #include "llvm/CodeGen/LivePhysRegs.h" | |
| #include "llvm/CodeGen/MachineBasicBlock.h" | |
| #include "llvm/CodeGen/MachineConstantPool.h" | |
| #include "llvm/CodeGen/MachineFunction.h" | |
| #include "llvm/CodeGen/MachineFunctionPass.h" | |
| #include "llvm/CodeGen/MachineInstr.h" | |
| #include "llvm/CodeGen/MachineInstrBuilder.h" | |
| #include "llvm/CodeGen/MachineOperand.h" | |
| #include "llvm/CodeGen/TargetSubtargetInfo.h" | |
| #include "llvm/IR/DebugLoc.h" | |
| #include "llvm/MC/MCInstrDesc.h" | |
| #include "llvm/Pass.h" | |
| #include "llvm/Support/CodeGen.h" | |
| #include "llvm/Support/MathExtras.h" | |
| #include "llvm/Target/TargetMachine.h" | |
| #include <cassert> | |
| #include <cstdint> | |
| #include <iterator> | |
| #include <limits> | |
| #include <utility> | |
| using namespace llvm; | |
| #define AARCH64_EXPAND_PSEUDO_NAME "AArch64 pseudo instruction expansion pass" | |
| namespace { | |
| class AArch64ExpandPseudo : public MachineFunctionPass { | |
| public: | |
| const AArch64InstrInfo *TII; | |
| static char ID; | |
| AArch64ExpandPseudo() : MachineFunctionPass(ID) { | |
| initializeAArch64ExpandPseudoPass(*PassRegistry::getPassRegistry()); | |
| } | |
| bool runOnMachineFunction(MachineFunction &Fn) override; | |
| StringRef getPassName() const override { return AARCH64_EXPAND_PSEUDO_NAME; } | |
| private: | |
| bool expandMBB(MachineBasicBlock &MBB); | |
| bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, | |
| MachineBasicBlock::iterator &NextMBBI); | |
| bool expandMOVImm(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, | |
| unsigned BitSize); | |
| bool expand_DestructiveOp(MachineInstr &MI, MachineBasicBlock &MBB, | |
| MachineBasicBlock::iterator MBBI); | |
| bool expandCMP_SWAP(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, | |
| unsigned LdarOp, unsigned StlrOp, unsigned CmpOp, | |
| unsigned ExtendImm, unsigned ZeroReg, | |
| MachineBasicBlock::iterator &NextMBBI); | |
| bool expandCMP_SWAP_128(MachineBasicBlock &MBB, | |
| MachineBasicBlock::iterator MBBI, | |
| MachineBasicBlock::iterator &NextMBBI); | |
| bool expandSetTagLoop(MachineBasicBlock &MBB, | |
| MachineBasicBlock::iterator MBBI, | |
| MachineBasicBlock::iterator &NextMBBI); | |
| bool expandSVESpillFill(MachineBasicBlock &MBB, | |
| MachineBasicBlock::iterator MBBI, unsigned Opc, | |
| unsigned N); | |
| bool expandCALL_RVMARKER(MachineBasicBlock &MBB, | |
| MachineBasicBlock::iterator MBBI); | |
| bool expandCALL_BTI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI); | |
| bool expandStoreSwiftAsyncContext(MachineBasicBlock &MBB, | |
| MachineBasicBlock::iterator MBBI); | |
| MachineBasicBlock *expandRestoreZA(MachineBasicBlock &MBB, | |
| MachineBasicBlock::iterator MBBI); | |
| MachineBasicBlock *expandCondSMToggle(MachineBasicBlock &MBB, | |
| MachineBasicBlock::iterator MBBI); | |
| }; | |
| } // end anonymous namespace | |
| char AArch64ExpandPseudo::ID = 0; | |
| INITIALIZE_PASS(AArch64ExpandPseudo, "aarch64-expand-pseudo", | |
| AARCH64_EXPAND_PSEUDO_NAME, false, false) | |
| /// Transfer implicit operands on the pseudo instruction to the | |
| /// instructions created from the expansion. | |
| static void transferImpOps(MachineInstr &OldMI, MachineInstrBuilder &UseMI, | |
| MachineInstrBuilder &DefMI) { | |
| const MCInstrDesc &Desc = OldMI.getDesc(); | |
| for (const MachineOperand &MO : | |
| llvm::drop_begin(OldMI.operands(), Desc.getNumOperands())) { | |
| assert(MO.isReg() && MO.getReg()); | |
| if (MO.isUse()) | |
| UseMI.add(MO); | |
| else | |
| DefMI.add(MO); | |
| } | |
| } | |
| /// Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more | |
| /// real move-immediate instructions to synthesize the immediate. | |
| bool AArch64ExpandPseudo::expandMOVImm(MachineBasicBlock &MBB, | |
| MachineBasicBlock::iterator MBBI, | |
| unsigned BitSize) { | |
| MachineInstr &MI = *MBBI; | |
| Register DstReg = MI.getOperand(0).getReg(); | |
| uint64_t RenamableState = | |
| MI.getOperand(0).isRenamable() ? RegState::Renamable : 0; | |
| uint64_t Imm = MI.getOperand(1).getImm(); | |
| if (DstReg == AArch64::XZR || DstReg == AArch64::WZR) { | |
| // Useless def, and we don't want to risk creating an invalid ORR (which | |
| // would really write to sp). | |
| MI.eraseFromParent(); | |
| return true; | |
| } | |
| SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn; | |
| AArch64_IMM::expandMOVImm(Imm, BitSize, Insn); | |
| assert(Insn.size() != 0); | |
| SmallVector<MachineInstrBuilder, 4> MIBS; | |
| for (auto I = Insn.begin(), E = Insn.end(); I != E; ++I) { | |
| bool LastItem = std::next(I) == E; | |
| switch (I->Opcode) | |
| { | |
| default: llvm_unreachable("unhandled!"); break; | |
| case AArch64::ORRWri: | |
| case AArch64::ORRXri: | |
| MIBS.push_back(BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(I->Opcode)) | |
| .add(MI.getOperand(0)) | |
| .addReg(BitSize == 32 ? AArch64::WZR : AArch64::XZR) | |
| .addImm(I->Op2)); | |
| break; | |
| case AArch64::MOVNWi: | |
| case AArch64::MOVNXi: | |
| case AArch64::MOVZWi: | |
| case AArch64::MOVZXi: { | |
| bool DstIsDead = MI.getOperand(0).isDead(); | |
| MIBS.push_back(BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(I->Opcode)) | |
| .addReg(DstReg, RegState::Define | | |
| getDeadRegState(DstIsDead && LastItem) | | |
| RenamableState) | |
| .addImm(I->Op1) | |
| .addImm(I->Op2)); | |
| } break; | |
| case AArch64::MOVKWi: | |
| case AArch64::MOVKXi: { | |
| Register DstReg = MI.getOperand(0).getReg(); | |
| bool DstIsDead = MI.getOperand(0).isDead(); | |
| MIBS.push_back(BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(I->Opcode)) | |
| .addReg(DstReg, | |
| RegState::Define | | |
| getDeadRegState(DstIsDead && LastItem) | | |
| RenamableState) | |
| .addReg(DstReg) | |
| .addImm(I->Op1) | |
| .addImm(I->Op2)); | |
| } break; | |
| } | |
| } | |
| transferImpOps(MI, MIBS.front(), MIBS.back()); | |
| MI.eraseFromParent(); | |
| return true; | |
| } | |
| bool AArch64ExpandPseudo::expandCMP_SWAP( | |
| MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned LdarOp, | |
| unsigned StlrOp, unsigned CmpOp, unsigned ExtendImm, unsigned ZeroReg, | |
| MachineBasicBlock::iterator &NextMBBI) { | |
| MachineInstr &MI = *MBBI; | |
| MIMetadata MIMD(MI); | |
| const MachineOperand &Dest = MI.getOperand(0); | |
| Register StatusReg = MI.getOperand(1).getReg(); | |
| bool StatusDead = MI.getOperand(1).isDead(); | |
| // Duplicating undef operands into 2 instructions does not guarantee the same | |
| // value on both; However undef should be replaced by xzr anyway. | |
| assert(!MI.getOperand(2).isUndef() && "cannot handle undef"); | |
| Register AddrReg = MI.getOperand(2).getReg(); | |
| Register DesiredReg = MI.getOperand(3).getReg(); | |
| Register NewReg = MI.getOperand(4).getReg(); | |
| MachineFunction *MF = MBB.getParent(); | |
| auto LoadCmpBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); | |
| auto StoreBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); | |
| auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); | |
| MF->insert(++MBB.getIterator(), LoadCmpBB); | |
| MF->insert(++LoadCmpBB->getIterator(), StoreBB); | |
| MF->insert(++StoreBB->getIterator(), DoneBB); | |
| // .Lloadcmp: | |
| // mov wStatus, 0 | |
| // ldaxr xDest, [xAddr] | |
| // cmp xDest, xDesired | |
| // b.ne .Ldone | |
| if (!StatusDead) | |
| BuildMI(LoadCmpBB, MIMD, TII->get(AArch64::MOVZWi), StatusReg) | |
| .addImm(0).addImm(0); | |
| BuildMI(LoadCmpBB, MIMD, TII->get(LdarOp), Dest.getReg()) | |
| .addReg(AddrReg); | |
| BuildMI(LoadCmpBB, MIMD, TII->get(CmpOp), ZeroReg) | |
| .addReg(Dest.getReg(), getKillRegState(Dest.isDead())) | |
| .addReg(DesiredReg) | |
| .addImm(ExtendImm); | |
| BuildMI(LoadCmpBB, MIMD, TII->get(AArch64::Bcc)) | |
| .addImm(AArch64CC::NE) | |
| .addMBB(DoneBB) | |
| .addReg(AArch64::NZCV, RegState::Implicit | RegState::Kill); | |
| LoadCmpBB->addSuccessor(DoneBB); | |
| LoadCmpBB->addSuccessor(StoreBB); | |
| // .Lstore: | |
| // stlxr wStatus, xNew, [xAddr] | |
| // cbnz wStatus, .Lloadcmp | |
| BuildMI(StoreBB, MIMD, TII->get(StlrOp), StatusReg) | |
| .addReg(NewReg) | |
| .addReg(AddrReg); | |
| BuildMI(StoreBB, MIMD, TII->get(AArch64::CBNZW)) | |
| .addReg(StatusReg, getKillRegState(StatusDead)) | |
| .addMBB(LoadCmpBB); | |
| StoreBB->addSuccessor(LoadCmpBB); | |
| StoreBB->addSuccessor(DoneBB); | |
| DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end()); | |
| DoneBB->transferSuccessors(&MBB); | |
| MBB.addSuccessor(LoadCmpBB); | |
| NextMBBI = MBB.end(); | |
| MI.eraseFromParent(); | |
| // Recompute livein lists. | |
| LivePhysRegs LiveRegs; | |
| computeAndAddLiveIns(LiveRegs, *DoneBB); | |
| computeAndAddLiveIns(LiveRegs, *StoreBB); | |
| computeAndAddLiveIns(LiveRegs, *LoadCmpBB); | |
| // Do an extra pass around the loop to get loop carried registers right. | |
| StoreBB->clearLiveIns(); | |
| computeAndAddLiveIns(LiveRegs, *StoreBB); | |
| LoadCmpBB->clearLiveIns(); | |
| computeAndAddLiveIns(LiveRegs, *LoadCmpBB); | |
| return true; | |
| } | |
| bool AArch64ExpandPseudo::expandCMP_SWAP_128( | |
| MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, | |
| MachineBasicBlock::iterator &NextMBBI) { | |
| MachineInstr &MI = *MBBI; | |
| MIMetadata MIMD(MI); | |
| MachineOperand &DestLo = MI.getOperand(0); | |
| MachineOperand &DestHi = MI.getOperand(1); | |
| Register StatusReg = MI.getOperand(2).getReg(); | |
| bool StatusDead = MI.getOperand(2).isDead(); | |
| // Duplicating undef operands into 2 instructions does not guarantee the same | |
| // value on both; However undef should be replaced by xzr anyway. | |
| assert(!MI.getOperand(3).isUndef() && "cannot handle undef"); | |
| Register AddrReg = MI.getOperand(3).getReg(); | |
| Register DesiredLoReg = MI.getOperand(4).getReg(); | |
| Register DesiredHiReg = MI.getOperand(5).getReg(); | |
| Register NewLoReg = MI.getOperand(6).getReg(); | |
| Register NewHiReg = MI.getOperand(7).getReg(); | |
| unsigned LdxpOp, StxpOp; | |
| switch (MI.getOpcode()) { | |
| case AArch64::CMP_SWAP_128_MONOTONIC: | |
| LdxpOp = AArch64::LDXPX; | |
| StxpOp = AArch64::STXPX; | |
| break; | |
| case AArch64::CMP_SWAP_128_RELEASE: | |
| LdxpOp = AArch64::LDXPX; | |
| StxpOp = AArch64::STLXPX; | |
| break; | |
| case AArch64::CMP_SWAP_128_ACQUIRE: | |
| LdxpOp = AArch64::LDAXPX; | |
| StxpOp = AArch64::STXPX; | |
| break; | |
| case AArch64::CMP_SWAP_128: | |
| LdxpOp = AArch64::LDAXPX; | |
| StxpOp = AArch64::STLXPX; | |
| break; | |
| default: | |
| llvm_unreachable("Unexpected opcode"); | |
| } | |
| MachineFunction *MF = MBB.getParent(); | |
| auto LoadCmpBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); | |
| auto StoreBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); | |
| auto FailBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); | |
| auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); | |
| MF->insert(++MBB.getIterator(), LoadCmpBB); | |
| MF->insert(++LoadCmpBB->getIterator(), StoreBB); | |
| MF->insert(++StoreBB->getIterator(), FailBB); | |
| MF->insert(++FailBB->getIterator(), DoneBB); | |
| // .Lloadcmp: | |
| // ldaxp xDestLo, xDestHi, [xAddr] | |
| // cmp xDestLo, xDesiredLo | |
| // sbcs xDestHi, xDesiredHi | |
| // b.ne .Ldone | |
| BuildMI(LoadCmpBB, MIMD, TII->get(LdxpOp)) | |
| .addReg(DestLo.getReg(), RegState::Define) | |
| .addReg(DestHi.getReg(), RegState::Define) | |
| .addReg(AddrReg); | |
| BuildMI(LoadCmpBB, MIMD, TII->get(AArch64::SUBSXrs), AArch64::XZR) | |
| .addReg(DestLo.getReg(), getKillRegState(DestLo.isDead())) | |
| .addReg(DesiredLoReg) | |
| .addImm(0); | |
| BuildMI(LoadCmpBB, MIMD, TII->get(AArch64::CSINCWr), StatusReg) | |
| .addUse(AArch64::WZR) | |
| .addUse(AArch64::WZR) | |
| .addImm(AArch64CC::EQ); | |
| BuildMI(LoadCmpBB, MIMD, TII->get(AArch64::SUBSXrs), AArch64::XZR) | |
| .addReg(DestHi.getReg(), getKillRegState(DestHi.isDead())) | |
| .addReg(DesiredHiReg) | |
| .addImm(0); | |
| BuildMI(LoadCmpBB, MIMD, TII->get(AArch64::CSINCWr), StatusReg) | |
| .addUse(StatusReg, RegState::Kill) | |
| .addUse(StatusReg, RegState::Kill) | |
| .addImm(AArch64CC::EQ); | |
| BuildMI(LoadCmpBB, MIMD, TII->get(AArch64::CBNZW)) | |
| .addUse(StatusReg, getKillRegState(StatusDead)) | |
| .addMBB(FailBB); | |
| LoadCmpBB->addSuccessor(FailBB); | |
| LoadCmpBB->addSuccessor(StoreBB); | |
| // .Lstore: | |
| // stlxp wStatus, xNewLo, xNewHi, [xAddr] | |
| // cbnz wStatus, .Lloadcmp | |
| BuildMI(StoreBB, MIMD, TII->get(StxpOp), StatusReg) | |
| .addReg(NewLoReg) | |
| .addReg(NewHiReg) | |
| .addReg(AddrReg); | |
| BuildMI(StoreBB, MIMD, TII->get(AArch64::CBNZW)) | |
| .addReg(StatusReg, getKillRegState(StatusDead)) | |
| .addMBB(LoadCmpBB); | |
| BuildMI(StoreBB, MIMD, TII->get(AArch64::B)).addMBB(DoneBB); | |
| StoreBB->addSuccessor(LoadCmpBB); | |
| StoreBB->addSuccessor(DoneBB); | |
| // .Lfail: | |
| // stlxp wStatus, xDestLo, xDestHi, [xAddr] | |
| // cbnz wStatus, .Lloadcmp | |
| BuildMI(FailBB, MIMD, TII->get(StxpOp), StatusReg) | |
| .addReg(DestLo.getReg()) | |
| .addReg(DestHi.getReg()) | |
| .addReg(AddrReg); | |
| BuildMI(FailBB, MIMD, TII->get(AArch64::CBNZW)) | |
| .addReg(StatusReg, getKillRegState(StatusDead)) | |
| .addMBB(LoadCmpBB); | |
| FailBB->addSuccessor(LoadCmpBB); | |
| FailBB->addSuccessor(DoneBB); | |
| DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end()); | |
| DoneBB->transferSuccessors(&MBB); | |
| MBB.addSuccessor(LoadCmpBB); | |
| NextMBBI = MBB.end(); | |
| MI.eraseFromParent(); | |
| // Recompute liveness bottom up. | |
| LivePhysRegs LiveRegs; | |
| computeAndAddLiveIns(LiveRegs, *DoneBB); | |
| computeAndAddLiveIns(LiveRegs, *FailBB); | |
| computeAndAddLiveIns(LiveRegs, *StoreBB); | |
| computeAndAddLiveIns(LiveRegs, *LoadCmpBB); | |
| // Do an extra pass in the loop to get the loop carried dependencies right. | |
| FailBB->clearLiveIns(); | |
| computeAndAddLiveIns(LiveRegs, *FailBB); | |
| StoreBB->clearLiveIns(); | |
| computeAndAddLiveIns(LiveRegs, *StoreBB); | |
| LoadCmpBB->clearLiveIns(); | |
| computeAndAddLiveIns(LiveRegs, *LoadCmpBB); | |
| return true; | |
| } | |
| /// \brief Expand Pseudos to Instructions with destructive operands. | |
| /// | |
| /// This mechanism uses MOVPRFX instructions for zeroing the false lanes | |
| /// or for fixing relaxed register allocation conditions to comply with | |
| /// the instructions register constraints. The latter case may be cheaper | |
| /// than setting the register constraints in the register allocator, | |
| /// since that will insert regular MOV instructions rather than MOVPRFX. | |
| /// | |
| /// Example (after register allocation): | |
| /// | |
| /// FSUB_ZPZZ_ZERO_B Z0, Pg, Z1, Z0 | |
| /// | |
| /// * The Pseudo FSUB_ZPZZ_ZERO_B maps to FSUB_ZPmZ_B. | |
| /// * We cannot map directly to FSUB_ZPmZ_B because the register | |
| /// constraints of the instruction are not met. | |
| /// * Also the _ZERO specifies the false lanes need to be zeroed. | |
| /// | |
| /// We first try to see if the destructive operand == result operand, | |
| /// if not, we try to swap the operands, e.g. | |
| /// | |
| /// FSUB_ZPmZ_B Z0, Pg/m, Z0, Z1 | |
| /// | |
| /// But because FSUB_ZPmZ is not commutative, this is semantically | |
| /// different, so we need a reverse instruction: | |
| /// | |
| /// FSUBR_ZPmZ_B Z0, Pg/m, Z0, Z1 | |
| /// | |
| /// Then we implement the zeroing of the false lanes of Z0 by adding | |
| /// a zeroing MOVPRFX instruction: | |
| /// | |
| /// MOVPRFX_ZPzZ_B Z0, Pg/z, Z0 | |
| /// FSUBR_ZPmZ_B Z0, Pg/m, Z0, Z1 | |
| /// | |
| /// Note that this can only be done for _ZERO or _UNDEF variants where | |
| /// we can guarantee the false lanes to be zeroed (by implementing this) | |
| /// or that they are undef (don't care / not used), otherwise the | |
| /// swapping of operands is illegal because the operation is not | |
| /// (or cannot be emulated to be) fully commutative. | |
| bool AArch64ExpandPseudo::expand_DestructiveOp( | |
| MachineInstr &MI, | |
| MachineBasicBlock &MBB, | |
| MachineBasicBlock::iterator MBBI) { | |
| unsigned Opcode = AArch64::getSVEPseudoMap(MI.getOpcode()); | |
| uint64_t DType = TII->get(Opcode).TSFlags & AArch64::DestructiveInstTypeMask; | |
| uint64_t FalseLanes = MI.getDesc().TSFlags & AArch64::FalseLanesMask; | |
| bool FalseZero = FalseLanes == AArch64::FalseLanesZero; | |
| Register DstReg = MI.getOperand(0).getReg(); | |
| bool DstIsDead = MI.getOperand(0).isDead(); | |
| bool UseRev = false; | |
| unsigned PredIdx, DOPIdx, SrcIdx, Src2Idx; | |
| switch (DType) { | |
| case AArch64::DestructiveBinaryComm: | |
| case AArch64::DestructiveBinaryCommWithRev: | |
| if (DstReg == MI.getOperand(3).getReg()) { | |
| // FSUB Zd, Pg, Zs1, Zd ==> FSUBR Zd, Pg/m, Zd, Zs1 | |
| std::tie(PredIdx, DOPIdx, SrcIdx) = std::make_tuple(1, 3, 2); | |
| UseRev = true; | |
| break; | |
| } | |
| [[fallthrough]]; | |
| case AArch64::DestructiveBinary: | |
| case AArch64::DestructiveBinaryImm: | |
| std::tie(PredIdx, DOPIdx, SrcIdx) = std::make_tuple(1, 2, 3); | |
| break; | |
| case AArch64::DestructiveUnaryPassthru: | |
| std::tie(PredIdx, DOPIdx, SrcIdx) = std::make_tuple(2, 3, 3); | |
| break; | |
| case AArch64::DestructiveTernaryCommWithRev: | |
| std::tie(PredIdx, DOPIdx, SrcIdx, Src2Idx) = std::make_tuple(1, 2, 3, 4); | |
| if (DstReg == MI.getOperand(3).getReg()) { | |
| // FMLA Zd, Pg, Za, Zd, Zm ==> FMAD Zdn, Pg, Zm, Za | |
| std::tie(PredIdx, DOPIdx, SrcIdx, Src2Idx) = std::make_tuple(1, 3, 4, 2); | |
| UseRev = true; | |
| } else if (DstReg == MI.getOperand(4).getReg()) { | |
| // FMLA Zd, Pg, Za, Zm, Zd ==> FMAD Zdn, Pg, Zm, Za | |
| std::tie(PredIdx, DOPIdx, SrcIdx, Src2Idx) = std::make_tuple(1, 4, 3, 2); | |
| UseRev = true; | |
| } | |
| break; | |
| default: | |
| llvm_unreachable("Unsupported Destructive Operand type"); | |
| } | |
| // MOVPRFX can only be used if the destination operand | |
| // is the destructive operand, not as any other operand, | |
| // so the Destructive Operand must be unique. | |
| bool DOPRegIsUnique = false; | |
| switch (DType) { | |
| case AArch64::DestructiveBinary: | |
| DOPRegIsUnique = DstReg != MI.getOperand(SrcIdx).getReg(); | |
| break; | |
| case AArch64::DestructiveBinaryComm: | |
| case AArch64::DestructiveBinaryCommWithRev: | |
| DOPRegIsUnique = | |
| DstReg != MI.getOperand(DOPIdx).getReg() || | |
| MI.getOperand(DOPIdx).getReg() != MI.getOperand(SrcIdx).getReg(); | |
| break; | |
| case AArch64::DestructiveUnaryPassthru: | |
| case AArch64::DestructiveBinaryImm: | |
| DOPRegIsUnique = true; | |
| break; | |
| case AArch64::DestructiveTernaryCommWithRev: | |
| DOPRegIsUnique = | |
| DstReg != MI.getOperand(DOPIdx).getReg() || | |
| (MI.getOperand(DOPIdx).getReg() != MI.getOperand(SrcIdx).getReg() && | |
| MI.getOperand(DOPIdx).getReg() != MI.getOperand(Src2Idx).getReg()); | |
| break; | |
| } | |
| // Resolve the reverse opcode | |
| if (UseRev) { | |
| int NewOpcode; | |
| // e.g. DIV -> DIVR | |
| if ((NewOpcode = AArch64::getSVERevInstr(Opcode)) != -1) | |
| Opcode = NewOpcode; | |
| // e.g. DIVR -> DIV | |
| else if ((NewOpcode = AArch64::getSVENonRevInstr(Opcode)) != -1) | |
| Opcode = NewOpcode; | |
| } | |
| // Get the right MOVPRFX | |
| uint64_t ElementSize = TII->getElementSizeForOpcode(Opcode); | |
| unsigned MovPrfx, LSLZero, MovPrfxZero; | |
| switch (ElementSize) { | |
| case AArch64::ElementSizeNone: | |
| case AArch64::ElementSizeB: | |
| MovPrfx = AArch64::MOVPRFX_ZZ; | |
| LSLZero = AArch64::LSL_ZPmI_B; | |
| MovPrfxZero = AArch64::MOVPRFX_ZPzZ_B; | |
| break; | |
| case AArch64::ElementSizeH: | |
| MovPrfx = AArch64::MOVPRFX_ZZ; | |
| LSLZero = AArch64::LSL_ZPmI_H; | |
| MovPrfxZero = AArch64::MOVPRFX_ZPzZ_H; | |
| break; | |
| case AArch64::ElementSizeS: | |
| MovPrfx = AArch64::MOVPRFX_ZZ; | |
| LSLZero = AArch64::LSL_ZPmI_S; | |
| MovPrfxZero = AArch64::MOVPRFX_ZPzZ_S; | |
| break; | |
| case AArch64::ElementSizeD: | |
| MovPrfx = AArch64::MOVPRFX_ZZ; | |
| LSLZero = AArch64::LSL_ZPmI_D; | |
| MovPrfxZero = AArch64::MOVPRFX_ZPzZ_D; | |
| break; | |
| default: | |
| llvm_unreachable("Unsupported ElementSize"); | |
| } | |
| // | |
| // Create the destructive operation (if required) | |
| // | |
| MachineInstrBuilder PRFX, DOP; | |
| if (FalseZero) { | |
| // If we cannot prefix the requested instruction we'll instead emit a | |
| // prefixed_zeroing_mov for DestructiveBinary. | |
| assert((DOPRegIsUnique || DType == AArch64::DestructiveBinary || | |
| DType == AArch64::DestructiveBinaryComm) && | |
| "The destructive operand should be unique"); | |
| assert(ElementSize != AArch64::ElementSizeNone && | |
| "This instruction is unpredicated"); | |
| // Merge source operand into destination register | |
| PRFX = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(MovPrfxZero)) | |
| .addReg(DstReg, RegState::Define) | |
| .addReg(MI.getOperand(PredIdx).getReg()) | |
| .addReg(MI.getOperand(DOPIdx).getReg()); | |
| // After the movprfx, the destructive operand is same as Dst | |
| DOPIdx = 0; | |
| // Create the additional LSL to zero the lanes when the DstReg is not | |
| // unique. Zeros the lanes in z0 that aren't active in p0 with sequence | |
| // movprfx z0.b, p0/z, z0.b; lsl z0.b, p0/m, z0.b, #0; | |
| if ((DType == AArch64::DestructiveBinary || | |
| DType == AArch64::DestructiveBinaryComm) && | |
| !DOPRegIsUnique) { | |
| BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(LSLZero)) | |
| .addReg(DstReg, RegState::Define) | |
| .add(MI.getOperand(PredIdx)) | |
| .addReg(DstReg) | |
| .addImm(0); | |
| } | |
| } else if (DstReg != MI.getOperand(DOPIdx).getReg()) { | |
| assert(DOPRegIsUnique && "The destructive operand should be unique"); | |
| PRFX = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(MovPrfx)) | |
| .addReg(DstReg, RegState::Define) | |
| .addReg(MI.getOperand(DOPIdx).getReg()); | |
| DOPIdx = 0; | |
| } | |
| // | |
| // Create the destructive operation | |
| // | |
| DOP = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opcode)) | |
| .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead)); | |
| switch (DType) { | |
| case AArch64::DestructiveUnaryPassthru: | |
| DOP.addReg(MI.getOperand(DOPIdx).getReg(), RegState::Kill) | |
| .add(MI.getOperand(PredIdx)) | |
| .add(MI.getOperand(SrcIdx)); | |
| break; | |
| case AArch64::DestructiveBinary: | |
| case AArch64::DestructiveBinaryImm: | |
| case AArch64::DestructiveBinaryComm: | |
| case AArch64::DestructiveBinaryCommWithRev: | |
| DOP.add(MI.getOperand(PredIdx)) | |
| .addReg(MI.getOperand(DOPIdx).getReg(), RegState::Kill) | |
| .add(MI.getOperand(SrcIdx)); | |
| break; | |
| case AArch64::DestructiveTernaryCommWithRev: | |
| DOP.add(MI.getOperand(PredIdx)) | |
| .addReg(MI.getOperand(DOPIdx).getReg(), RegState::Kill) | |
| .add(MI.getOperand(SrcIdx)) | |
| .add(MI.getOperand(Src2Idx)); | |
| break; | |
| } | |
| if (PRFX) { | |
| finalizeBundle(MBB, PRFX->getIterator(), MBBI->getIterator()); | |
| transferImpOps(MI, PRFX, DOP); | |
| } else | |
| transferImpOps(MI, DOP, DOP); | |
| MI.eraseFromParent(); | |
| return true; | |
| } | |
| bool AArch64ExpandPseudo::expandSetTagLoop( | |
| MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, | |
| MachineBasicBlock::iterator &NextMBBI) { | |
| MachineInstr &MI = *MBBI; | |
| DebugLoc DL = MI.getDebugLoc(); | |
| Register SizeReg = MI.getOperand(0).getReg(); | |
| Register AddressReg = MI.getOperand(1).getReg(); | |
| MachineFunction *MF = MBB.getParent(); | |
| bool ZeroData = MI.getOpcode() == AArch64::STZGloop_wback; | |
| const unsigned OpCode1 = | |
| ZeroData ? AArch64::STZGPostIndex : AArch64::STGPostIndex; | |
| const unsigned OpCode2 = | |
| ZeroData ? AArch64::STZ2GPostIndex : AArch64::ST2GPostIndex; | |
| unsigned Size = MI.getOperand(2).getImm(); | |
| assert(Size > 0 && Size % 16 == 0); | |
| if (Size % (16 * 2) != 0) { | |
| BuildMI(MBB, MBBI, DL, TII->get(OpCode1), AddressReg) | |
| .addReg(AddressReg) | |
| .addReg(AddressReg) | |
| .addImm(1); | |
| Size -= 16; | |
| } | |
| MachineBasicBlock::iterator I = | |
| BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm), SizeReg) | |
| .addImm(Size); | |
| expandMOVImm(MBB, I, 64); | |
| auto LoopBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); | |
| auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); | |
| MF->insert(++MBB.getIterator(), LoopBB); | |
| MF->insert(++LoopBB->getIterator(), DoneBB); | |
| BuildMI(LoopBB, DL, TII->get(OpCode2)) | |
| .addDef(AddressReg) | |
| .addReg(AddressReg) | |
| .addReg(AddressReg) | |
| .addImm(2) | |
| .cloneMemRefs(MI) | |
| .setMIFlags(MI.getFlags()); | |
| BuildMI(LoopBB, DL, TII->get(AArch64::SUBXri)) | |
| .addDef(SizeReg) | |
| .addReg(SizeReg) | |
| .addImm(16 * 2) | |
| .addImm(0); | |
| BuildMI(LoopBB, DL, TII->get(AArch64::CBNZX)).addUse(SizeReg).addMBB(LoopBB); | |
| LoopBB->addSuccessor(LoopBB); | |
| LoopBB->addSuccessor(DoneBB); | |
| DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end()); | |
| DoneBB->transferSuccessors(&MBB); | |
| MBB.addSuccessor(LoopBB); | |
| NextMBBI = MBB.end(); | |
| MI.eraseFromParent(); | |
| // Recompute liveness bottom up. | |
| LivePhysRegs LiveRegs; | |
| computeAndAddLiveIns(LiveRegs, *DoneBB); | |
| computeAndAddLiveIns(LiveRegs, *LoopBB); | |
| // Do an extra pass in the loop to get the loop carried dependencies right. | |
| // FIXME: is this necessary? | |
| LoopBB->clearLiveIns(); | |
| computeAndAddLiveIns(LiveRegs, *LoopBB); | |
| DoneBB->clearLiveIns(); | |
| computeAndAddLiveIns(LiveRegs, *DoneBB); | |
| return true; | |
| } | |
| bool AArch64ExpandPseudo::expandSVESpillFill(MachineBasicBlock &MBB, | |
| MachineBasicBlock::iterator MBBI, | |
| unsigned Opc, unsigned N) { | |
| const TargetRegisterInfo *TRI = | |
| MBB.getParent()->getSubtarget().getRegisterInfo(); | |
| MachineInstr &MI = *MBBI; | |
| for (unsigned Offset = 0; Offset < N; ++Offset) { | |
| int ImmOffset = MI.getOperand(2).getImm() + Offset; | |
| bool Kill = (Offset + 1 == N) ? MI.getOperand(1).isKill() : false; | |
| assert(ImmOffset >= -256 && ImmOffset < 256 && | |
| "Immediate spill offset out of range"); | |
| BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc)) | |
| .addReg( | |
| TRI->getSubReg(MI.getOperand(0).getReg(), AArch64::zsub0 + Offset), | |
| Opc == AArch64::LDR_ZXI ? RegState::Define : 0) | |
| .addReg(MI.getOperand(1).getReg(), getKillRegState(Kill)) | |
| .addImm(ImmOffset); | |
| } | |
| MI.eraseFromParent(); | |
| return true; | |
| } | |
| bool AArch64ExpandPseudo::expandCALL_RVMARKER( | |
| MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) { | |
| // Expand CALL_RVMARKER pseudo to: | |
| // - a branch to the call target, followed by | |
| // - the special `mov x29, x29` marker, and | |
| // - another branch, to the runtime function | |
| // Mark the sequence as bundle, to avoid passes moving other code in between. | |
| MachineInstr &MI = *MBBI; | |
| MachineInstr *OriginalCall; | |
| MachineOperand &RVTarget = MI.getOperand(0); | |
| MachineOperand &CallTarget = MI.getOperand(1); | |
| assert((CallTarget.isGlobal() || CallTarget.isReg()) && | |
| "invalid operand for regular call"); | |
| assert(RVTarget.isGlobal() && "invalid operand for attached call"); | |
| unsigned Opc = CallTarget.isGlobal() ? AArch64::BL : AArch64::BLR; | |
| OriginalCall = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc)).getInstr(); | |
| OriginalCall->addOperand(CallTarget); | |
| unsigned RegMaskStartIdx = 2; | |
| // Skip register arguments. Those are added during ISel, but are not | |
| // needed for the concrete branch. | |
| while (!MI.getOperand(RegMaskStartIdx).isRegMask()) { | |
| auto MOP = MI.getOperand(RegMaskStartIdx); | |
| assert(MOP.isReg() && "can only add register operands"); | |
| OriginalCall->addOperand(MachineOperand::CreateReg( | |
| MOP.getReg(), /*Def=*/false, /*Implicit=*/true)); | |
| RegMaskStartIdx++; | |
| } | |
| for (const MachineOperand &MO : | |
| llvm::drop_begin(MI.operands(), RegMaskStartIdx)) | |
| OriginalCall->addOperand(MO); | |
| BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ORRXrs)) | |
| .addReg(AArch64::FP, RegState::Define) | |
| .addReg(AArch64::XZR) | |
| .addReg(AArch64::FP) | |
| .addImm(0); | |
| auto *RVCall = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::BL)) | |
| .add(RVTarget) | |
| .getInstr(); | |
| if (MI.shouldUpdateCallSiteInfo()) | |
| MBB.getParent()->moveCallSiteInfo(&MI, OriginalCall); | |
| MI.eraseFromParent(); | |
| finalizeBundle(MBB, OriginalCall->getIterator(), | |
| std::next(RVCall->getIterator())); | |
| return true; | |
| } | |
| bool AArch64ExpandPseudo::expandCALL_BTI(MachineBasicBlock &MBB, | |
| MachineBasicBlock::iterator MBBI) { | |
| // Expand CALL_BTI pseudo to: | |
| // - a branch to the call target | |
| // - a BTI instruction | |
| // Mark the sequence as a bundle, to avoid passes moving other code in | |
| // between. | |
| MachineInstr &MI = *MBBI; | |
| MachineOperand &CallTarget = MI.getOperand(0); | |
| assert((CallTarget.isGlobal() || CallTarget.isReg()) && | |
| "invalid operand for regular call"); | |
| unsigned Opc = CallTarget.isGlobal() ? AArch64::BL : AArch64::BLR; | |
| MachineInstr *Call = | |
| BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc)).getInstr(); | |
| Call->addOperand(CallTarget); | |
| Call->setCFIType(*MBB.getParent(), MI.getCFIType()); | |
| MachineInstr *BTI = | |
| BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::HINT)) | |
| // BTI J so that setjmp can to BR to this. | |
| .addImm(36) | |
| .getInstr(); | |
| if (MI.shouldUpdateCallSiteInfo()) | |
| MBB.getParent()->moveCallSiteInfo(&MI, Call); | |
| MI.eraseFromParent(); | |
| finalizeBundle(MBB, Call->getIterator(), std::next(BTI->getIterator())); | |
| return true; | |
| } | |
| bool AArch64ExpandPseudo::expandStoreSwiftAsyncContext( | |
| MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) { | |
| Register CtxReg = MBBI->getOperand(0).getReg(); | |
| Register BaseReg = MBBI->getOperand(1).getReg(); | |
| int Offset = MBBI->getOperand(2).getImm(); | |
| DebugLoc DL(MBBI->getDebugLoc()); | |
| auto &STI = MBB.getParent()->getSubtarget<AArch64Subtarget>(); | |
| if (STI.getTargetTriple().getArchName() != "arm64e") { | |
| BuildMI(MBB, MBBI, DL, TII->get(AArch64::STRXui)) | |
| .addUse(CtxReg) | |
| .addUse(BaseReg) | |
| .addImm(Offset / 8) | |
| .setMIFlag(MachineInstr::FrameSetup); | |
| MBBI->eraseFromParent(); | |
| return true; | |
| } | |
| // We need to sign the context in an address-discriminated way. 0xc31a is a | |
| // fixed random value, chosen as part of the ABI. | |
| // add x16, xBase, #Offset | |
| // movk x16, #0xc31a, lsl #48 | |
| // mov x17, x22/xzr | |
| // pacdb x17, x16 | |
| // str x17, [xBase, #Offset] | |
| unsigned Opc = Offset >= 0 ? AArch64::ADDXri : AArch64::SUBXri; | |
| BuildMI(MBB, MBBI, DL, TII->get(Opc), AArch64::X16) | |
| .addUse(BaseReg) | |
| .addImm(abs(Offset)) | |
| .addImm(0) | |
| .setMIFlag(MachineInstr::FrameSetup); | |
| BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVKXi), AArch64::X16) | |
| .addUse(AArch64::X16) | |
| .addImm(0xc31a) | |
| .addImm(48) | |
| .setMIFlag(MachineInstr::FrameSetup); | |
| // We're not allowed to clobber X22 (and couldn't clobber XZR if we tried), so | |
| // move it somewhere before signing. | |
| BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrs), AArch64::X17) | |
| .addUse(AArch64::XZR) | |
| .addUse(CtxReg) | |
| .addImm(0) | |
| .setMIFlag(MachineInstr::FrameSetup); | |
| BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACDB), AArch64::X17) | |
| .addUse(AArch64::X17) | |
| .addUse(AArch64::X16) | |
| .setMIFlag(MachineInstr::FrameSetup); | |
| BuildMI(MBB, MBBI, DL, TII->get(AArch64::STRXui)) | |
| .addUse(AArch64::X17) | |
| .addUse(BaseReg) | |
| .addImm(Offset / 8) | |
| .setMIFlag(MachineInstr::FrameSetup); | |
| MBBI->eraseFromParent(); | |
| return true; | |
| } | |
| MachineBasicBlock * | |
| AArch64ExpandPseudo::expandRestoreZA(MachineBasicBlock &MBB, | |
| MachineBasicBlock::iterator MBBI) { | |
| MachineInstr &MI = *MBBI; | |
| assert((std::next(MBBI) != MBB.end() || | |
| MI.getParent()->successors().begin() != | |
| MI.getParent()->successors().end()) && | |
| "Unexpected unreachable in block that restores ZA"); | |
| // Compare TPIDR2_EL0 value against 0. | |
| DebugLoc DL = MI.getDebugLoc(); | |
| MachineInstrBuilder Cbz = BuildMI(MBB, MBBI, DL, TII->get(AArch64::CBZX)) | |
| .add(MI.getOperand(0)); | |
| // Split MBB and create two new blocks: | |
| // - MBB now contains all instructions before RestoreZAPseudo. | |
| // - SMBB contains the RestoreZAPseudo instruction only. | |
| // - EndBB contains all instructions after RestoreZAPseudo. | |
| MachineInstr &PrevMI = *std::prev(MBBI); | |
| MachineBasicBlock *SMBB = MBB.splitAt(PrevMI, /*UpdateLiveIns*/ true); | |
| MachineBasicBlock *EndBB = std::next(MI.getIterator()) == SMBB->end() | |
| ? *SMBB->successors().begin() | |
| : SMBB->splitAt(MI, /*UpdateLiveIns*/ true); | |
| // Add the SMBB label to the TB[N]Z instruction & create a branch to EndBB. | |
| Cbz.addMBB(SMBB); | |
| BuildMI(&MBB, DL, TII->get(AArch64::B)) | |
| .addMBB(EndBB); | |
| MBB.addSuccessor(EndBB); | |
| // Replace the pseudo with a call (BL). | |
| MachineInstrBuilder MIB = | |
| BuildMI(*SMBB, SMBB->end(), DL, TII->get(AArch64::BL)); | |
| MIB.addReg(MI.getOperand(1).getReg(), RegState::Implicit); | |
| for (unsigned I = 2; I < MI.getNumOperands(); ++I) | |
| MIB.add(MI.getOperand(I)); | |
| BuildMI(SMBB, DL, TII->get(AArch64::B)).addMBB(EndBB); | |
| MI.eraseFromParent(); | |
| return EndBB; | |
| } | |
| MachineBasicBlock * | |
| AArch64ExpandPseudo::expandCondSMToggle(MachineBasicBlock &MBB, | |
| MachineBasicBlock::iterator MBBI) { | |
| MachineInstr &MI = *MBBI; | |
| // In the case of a smstart/smstop before a unreachable, just remove the pseudo. | |
| // Exception handling code generated by Clang may introduce unreachables and it | |
| // seems unnecessary to restore pstate.sm when that happens. Note that it is | |
| // not just an optimisation, the code below expects a successor instruction/block | |
| // in order to split the block at MBBI. | |
| if (std::next(MBBI) == MBB.end() && | |
| MI.getParent()->successors().begin() == | |
| MI.getParent()->successors().end()) { | |
| MI.eraseFromParent(); | |
| return &MBB; | |
| } | |
| // Expand the pseudo into smstart or smstop instruction. The pseudo has the | |
| // following operands: | |
| // | |
| // MSRpstatePseudo <za|sm|both>, <0|1>, pstate.sm, expectedval, <regmask> | |
| // | |
| // The pseudo is expanded into a conditional smstart/smstop, with a | |
| // check if pstate.sm (register) equals the expected value, and if not, | |
| // invokes the smstart/smstop. | |
| // | |
| // As an example, the following block contains a normal call from a | |
| // streaming-compatible function: | |
| // | |
| // OrigBB: | |
| // MSRpstatePseudo 3, 0, %0, 0, <regmask> <- Conditional SMSTOP | |
| // bl @normal_callee | |
| // MSRpstatePseudo 3, 1, %0, 0, <regmask> <- Conditional SMSTART | |
| // | |
| // ...which will be transformed into: | |
| // | |
| // OrigBB: | |
| // TBNZx %0:gpr64, 0, SMBB | |
| // b EndBB | |
| // | |
| // SMBB: | |
| // MSRpstatesvcrImm1 3, 0, <regmask> <- SMSTOP | |
| // | |
| // EndBB: | |
| // bl @normal_callee | |
| // MSRcond_pstatesvcrImm1 3, 1, <regmask> <- SMSTART | |
| // | |
| DebugLoc DL = MI.getDebugLoc(); | |
| // Create the conditional branch based on the third operand of the | |
| // instruction, which tells us if we are wrapping a normal or streaming | |
| // function. | |
| // We test the live value of pstate.sm and toggle pstate.sm if this is not the | |
| // expected value for the callee (0 for a normal callee and 1 for a streaming | |
| // callee). | |
| auto PStateSM = MI.getOperand(2).getReg(); | |
| bool IsStreamingCallee = MI.getOperand(3).getImm(); | |
| unsigned Opc = IsStreamingCallee ? AArch64::TBZX : AArch64::TBNZX; | |
| MachineInstrBuilder Tbx = | |
| BuildMI(MBB, MBBI, DL, TII->get(Opc)).addReg(PStateSM).addImm(0); | |
| // Split MBB and create two new blocks: | |
| // - MBB now contains all instructions before MSRcond_pstatesvcrImm1. | |
| // - SMBB contains the MSRcond_pstatesvcrImm1 instruction only. | |
| // - EndBB contains all instructions after MSRcond_pstatesvcrImm1. | |
| MachineInstr &PrevMI = *std::prev(MBBI); | |
| MachineBasicBlock *SMBB = MBB.splitAt(PrevMI, /*UpdateLiveIns*/ true); | |
| MachineBasicBlock *EndBB = std::next(MI.getIterator()) == SMBB->end() | |
| ? *SMBB->successors().begin() | |
| : SMBB->splitAt(MI, /*UpdateLiveIns*/ true); | |
| // Add the SMBB label to the TB[N]Z instruction & create a branch to EndBB. | |
| Tbx.addMBB(SMBB); | |
| BuildMI(&MBB, DL, TII->get(AArch64::B)) | |
| .addMBB(EndBB); | |
| MBB.addSuccessor(EndBB); | |
| // Create the SMSTART/SMSTOP (MSRpstatesvcrImm1) instruction in SMBB. | |
| MachineInstrBuilder MIB = BuildMI(*SMBB, SMBB->begin(), MI.getDebugLoc(), | |
| TII->get(AArch64::MSRpstatesvcrImm1)); | |
| // Copy all but the second and third operands of MSRcond_pstatesvcrImm1 (as | |
| // these contain the CopyFromReg for the first argument and the flag to | |
| // indicate whether the callee is streaming or normal). | |
| MIB.add(MI.getOperand(0)); | |
| MIB.add(MI.getOperand(1)); | |
| for (unsigned i = 4; i < MI.getNumOperands(); ++i) | |
| MIB.add(MI.getOperand(i)); | |
| BuildMI(SMBB, DL, TII->get(AArch64::B)).addMBB(EndBB); | |
| MI.eraseFromParent(); | |
| return EndBB; | |
| } | |
| /// If MBBI references a pseudo instruction that should be expanded here, | |
| /// do the expansion and return true. Otherwise return false. | |
| bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB, | |
| MachineBasicBlock::iterator MBBI, | |
| MachineBasicBlock::iterator &NextMBBI) { | |
| MachineInstr &MI = *MBBI; | |
| unsigned Opcode = MI.getOpcode(); | |
| // Check if we can expand the destructive op | |
| int OrigInstr = AArch64::getSVEPseudoMap(MI.getOpcode()); | |
| if (OrigInstr != -1) { | |
| auto &Orig = TII->get(OrigInstr); | |
| if ((Orig.TSFlags & AArch64::DestructiveInstTypeMask) | |
| != AArch64::NotDestructive) { | |
| return expand_DestructiveOp(MI, MBB, MBBI); | |
| } | |
| } | |
| switch (Opcode) { | |
| default: | |
| break; | |
| case AArch64::BSPv8i8: | |
| case AArch64::BSPv16i8: { | |
| Register DstReg = MI.getOperand(0).getReg(); | |
| if (DstReg == MI.getOperand(3).getReg()) { | |
| // Expand to BIT | |
| BuildMI(MBB, MBBI, MI.getDebugLoc(), | |
| TII->get(Opcode == AArch64::BSPv8i8 ? AArch64::BITv8i8 | |
| : AArch64::BITv16i8)) | |
| .add(MI.getOperand(0)) | |
| .add(MI.getOperand(3)) | |
| .add(MI.getOperand(2)) | |
| .add(MI.getOperand(1)); | |
| } else if (DstReg == MI.getOperand(2).getReg()) { | |
| // Expand to BIF | |
| BuildMI(MBB, MBBI, MI.getDebugLoc(), | |
| TII->get(Opcode == AArch64::BSPv8i8 ? AArch64::BIFv8i8 | |
| : AArch64::BIFv16i8)) | |
| .add(MI.getOperand(0)) | |
| .add(MI.getOperand(2)) | |
| .add(MI.getOperand(3)) | |
| .add(MI.getOperand(1)); | |
| } else { | |
| // Expand to BSL, use additional move if required | |
| if (DstReg == MI.getOperand(1).getReg()) { | |
| BuildMI(MBB, MBBI, MI.getDebugLoc(), | |
| TII->get(Opcode == AArch64::BSPv8i8 ? AArch64::BSLv8i8 | |
| : AArch64::BSLv16i8)) | |
| .add(MI.getOperand(0)) | |
| .add(MI.getOperand(1)) | |
| .add(MI.getOperand(2)) | |
| .add(MI.getOperand(3)); | |
| } else { | |
| BuildMI(MBB, MBBI, MI.getDebugLoc(), | |
| TII->get(Opcode == AArch64::BSPv8i8 ? AArch64::ORRv8i8 | |
| : AArch64::ORRv16i8)) | |
| .addReg(DstReg, | |
| RegState::Define | | |
| getRenamableRegState(MI.getOperand(0).isRenamable())) | |
| .add(MI.getOperand(1)) | |
| .add(MI.getOperand(1)); | |
| BuildMI(MBB, MBBI, MI.getDebugLoc(), | |
| TII->get(Opcode == AArch64::BSPv8i8 ? AArch64::BSLv8i8 | |
| : AArch64::BSLv16i8)) | |
| .add(MI.getOperand(0)) | |
| .addReg(DstReg, | |
| RegState::Kill | | |
| getRenamableRegState(MI.getOperand(0).isRenamable())) | |
| .add(MI.getOperand(2)) | |
| .add(MI.getOperand(3)); | |
| } | |
| } | |
| MI.eraseFromParent(); | |
| return true; | |
| } | |
| case AArch64::ADDWrr: | |
| case AArch64::SUBWrr: | |
| case AArch64::ADDXrr: | |
| case AArch64::SUBXrr: | |
| case AArch64::ADDSWrr: | |
| case AArch64::SUBSWrr: | |
| case AArch64::ADDSXrr: | |
| case AArch64::SUBSXrr: | |
| case AArch64::ANDWrr: | |
| case AArch64::ANDXrr: | |
| case AArch64::BICWrr: | |
| case AArch64::BICXrr: | |
| case AArch64::ANDSWrr: | |
| case AArch64::ANDSXrr: | |
| case AArch64::BICSWrr: | |
| case AArch64::BICSXrr: | |
| case AArch64::EONWrr: | |
| case AArch64::EONXrr: | |
| case AArch64::EORWrr: | |
| case AArch64::EORXrr: | |
| case AArch64::ORNWrr: | |
| case AArch64::ORNXrr: | |
| case AArch64::ORRWrr: | |
| case AArch64::ORRXrr: { | |
| unsigned Opcode; | |
| switch (MI.getOpcode()) { | |
| default: | |
| return false; | |
| case AArch64::ADDWrr: Opcode = AArch64::ADDWrs; break; | |
| case AArch64::SUBWrr: Opcode = AArch64::SUBWrs; break; | |
| case AArch64::ADDXrr: Opcode = AArch64::ADDXrs; break; | |
| case AArch64::SUBXrr: Opcode = AArch64::SUBXrs; break; | |
| case AArch64::ADDSWrr: Opcode = AArch64::ADDSWrs; break; | |
| case AArch64::SUBSWrr: Opcode = AArch64::SUBSWrs; break; | |
| case AArch64::ADDSXrr: Opcode = AArch64::ADDSXrs; break; | |
| case AArch64::SUBSXrr: Opcode = AArch64::SUBSXrs; break; | |
| case AArch64::ANDWrr: Opcode = AArch64::ANDWrs; break; | |
| case AArch64::ANDXrr: Opcode = AArch64::ANDXrs; break; | |
| case AArch64::BICWrr: Opcode = AArch64::BICWrs; break; | |
| case AArch64::BICXrr: Opcode = AArch64::BICXrs; break; | |
| case AArch64::ANDSWrr: Opcode = AArch64::ANDSWrs; break; | |
| case AArch64::ANDSXrr: Opcode = AArch64::ANDSXrs; break; | |
| case AArch64::BICSWrr: Opcode = AArch64::BICSWrs; break; | |
| case AArch64::BICSXrr: Opcode = AArch64::BICSXrs; break; | |
| case AArch64::EONWrr: Opcode = AArch64::EONWrs; break; | |
| case AArch64::EONXrr: Opcode = AArch64::EONXrs; break; | |
| case AArch64::EORWrr: Opcode = AArch64::EORWrs; break; | |
| case AArch64::EORXrr: Opcode = AArch64::EORXrs; break; | |
| case AArch64::ORNWrr: Opcode = AArch64::ORNWrs; break; | |
| case AArch64::ORNXrr: Opcode = AArch64::ORNXrs; break; | |
| case AArch64::ORRWrr: Opcode = AArch64::ORRWrs; break; | |
| case AArch64::ORRXrr: Opcode = AArch64::ORRXrs; break; | |
| } | |
| MachineFunction &MF = *MBB.getParent(); | |
| // Try to create new inst without implicit operands added. | |
| MachineInstr *NewMI = MF.CreateMachineInstr( | |
| TII->get(Opcode), MI.getDebugLoc(), /*NoImplicit=*/true); | |
| MBB.insert(MBBI, NewMI); | |
| MachineInstrBuilder MIB1(MF, NewMI); | |
| MIB1->setPCSections(MF, MI.getPCSections()); | |
| MIB1.addReg(MI.getOperand(0).getReg(), RegState::Define) | |
| .add(MI.getOperand(1)) | |
| .add(MI.getOperand(2)) | |
| .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0)); | |
| transferImpOps(MI, MIB1, MIB1); | |
| MI.eraseFromParent(); | |
| return true; | |
| } | |
| case AArch64::LOADgot: { | |
| MachineFunction *MF = MBB.getParent(); | |
| Register DstReg = MI.getOperand(0).getReg(); | |
| const MachineOperand &MO1 = MI.getOperand(1); | |
| unsigned Flags = MO1.getTargetFlags(); | |
| if (MF->getTarget().getCodeModel() == CodeModel::Tiny) { | |
| // Tiny codemodel expand to LDR | |
| MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(), | |
| TII->get(AArch64::LDRXl), DstReg); | |
| if (MO1.isGlobal()) { | |
| MIB.addGlobalAddress(MO1.getGlobal(), 0, Flags); | |
| } else if (MO1.isSymbol()) { | |
| MIB.addExternalSymbol(MO1.getSymbolName(), Flags); | |
| } else { | |
| assert(MO1.isCPI() && | |
| "Only expect globals, externalsymbols, or constant pools"); | |
| MIB.addConstantPoolIndex(MO1.getIndex(), MO1.getOffset(), Flags); | |
| } | |
| } else { | |
| // Small codemodel expand into ADRP + LDR. | |
| MachineFunction &MF = *MI.getParent()->getParent(); | |
| DebugLoc DL = MI.getDebugLoc(); | |
| MachineInstrBuilder MIB1 = | |
| BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADRP), DstReg); | |
| MachineInstrBuilder MIB2; | |
| if (MF.getSubtarget<AArch64Subtarget>().isTargetILP32()) { | |
| auto TRI = MBB.getParent()->getSubtarget().getRegisterInfo(); | |
| unsigned Reg32 = TRI->getSubReg(DstReg, AArch64::sub_32); | |
| unsigned DstFlags = MI.getOperand(0).getTargetFlags(); | |
| MIB2 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::LDRWui)) | |
| .addDef(Reg32) | |
| .addReg(DstReg, RegState::Kill) | |
| .addReg(DstReg, DstFlags | RegState::Implicit); | |
| } else { | |
| Register DstReg = MI.getOperand(0).getReg(); | |
| MIB2 = BuildMI(MBB, MBBI, DL, TII->get(AArch64::LDRXui)) | |
| .add(MI.getOperand(0)) | |
| .addUse(DstReg, RegState::Kill); | |
| } | |
| if (MO1.isGlobal()) { | |
| MIB1.addGlobalAddress(MO1.getGlobal(), 0, Flags | AArch64II::MO_PAGE); | |
| MIB2.addGlobalAddress(MO1.getGlobal(), 0, | |
| Flags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC); | |
| } else if (MO1.isSymbol()) { | |
| MIB1.addExternalSymbol(MO1.getSymbolName(), Flags | AArch64II::MO_PAGE); | |
| MIB2.addExternalSymbol(MO1.getSymbolName(), Flags | | |
| AArch64II::MO_PAGEOFF | | |
| AArch64II::MO_NC); | |
| } else { | |
| assert(MO1.isCPI() && | |
| "Only expect globals, externalsymbols, or constant pools"); | |
| MIB1.addConstantPoolIndex(MO1.getIndex(), MO1.getOffset(), | |
| Flags | AArch64II::MO_PAGE); | |
| MIB2.addConstantPoolIndex(MO1.getIndex(), MO1.getOffset(), | |
| Flags | AArch64II::MO_PAGEOFF | | |
| AArch64II::MO_NC); | |
| } | |
| transferImpOps(MI, MIB1, MIB2); | |
| } | |
| MI.eraseFromParent(); | |
| return true; | |
| } | |
| case AArch64::MOVaddrBA: { | |
| MachineFunction &MF = *MI.getParent()->getParent(); | |
| if (MF.getSubtarget<AArch64Subtarget>().isTargetMachO()) { | |
| // blockaddress expressions have to come from a constant pool because the | |
| // largest addend (and hence offset within a function) allowed for ADRP is | |
| // only 8MB. | |
| const BlockAddress *BA = MI.getOperand(1).getBlockAddress(); | |
| assert(MI.getOperand(1).getOffset() == 0 && "unexpected offset"); | |
| MachineConstantPool *MCP = MF.getConstantPool(); | |
| unsigned CPIdx = MCP->getConstantPoolIndex(BA, Align(8)); | |
| Register DstReg = MI.getOperand(0).getReg(); | |
| auto MIB1 = | |
| BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADRP), DstReg) | |
| .addConstantPoolIndex(CPIdx, 0, AArch64II::MO_PAGE); | |
| auto MIB2 = BuildMI(MBB, MBBI, MI.getDebugLoc(), | |
| TII->get(AArch64::LDRXui), DstReg) | |
| .addUse(DstReg) | |
| .addConstantPoolIndex( | |
| CPIdx, 0, AArch64II::MO_PAGEOFF | AArch64II::MO_NC); | |
| transferImpOps(MI, MIB1, MIB2); | |
| MI.eraseFromParent(); | |
| return true; | |
| } | |
| } | |
| [[fallthrough]]; | |
| case AArch64::MOVaddr: | |
| case AArch64::MOVaddrJT: | |
| case AArch64::MOVaddrCP: | |
| case AArch64::MOVaddrTLS: | |
| case AArch64::MOVaddrEXT: { | |
| // Expand into ADRP + ADD. | |
| Register DstReg = MI.getOperand(0).getReg(); | |
| assert(DstReg != AArch64::XZR); | |
| MachineInstrBuilder MIB1 = | |
| BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADRP), DstReg) | |
| .add(MI.getOperand(1)); | |
| if (MI.getOperand(1).getTargetFlags() & AArch64II::MO_TAGGED) { | |
| // MO_TAGGED on the page indicates a tagged address. Set the tag now. | |
| // We do so by creating a MOVK that sets bits 48-63 of the register to | |
| // (global address + 0x100000000 - PC) >> 48. This assumes that we're in | |
| // the small code model so we can assume a binary size of <= 4GB, which | |
| // makes the untagged PC relative offset positive. The binary must also be | |
| // loaded into address range [0, 2^48). Both of these properties need to | |
| // be ensured at runtime when using tagged addresses. | |
| auto Tag = MI.getOperand(1); | |
| Tag.setTargetFlags(AArch64II::MO_PREL | AArch64II::MO_G3); | |
| Tag.setOffset(0x100000000); | |
| BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::MOVKXi), DstReg) | |
| .addReg(DstReg) | |
| .add(Tag) | |
| .addImm(48); | |
| } | |
| MachineInstrBuilder MIB2 = | |
| BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADDXri)) | |
| .add(MI.getOperand(0)) | |
| .addReg(DstReg) | |
| .add(MI.getOperand(2)) | |
| .addImm(0); | |
| transferImpOps(MI, MIB1, MIB2); | |
| MI.eraseFromParent(); | |
| return true; | |
| } | |
| case AArch64::ADDlowTLS: | |
| // Produce a plain ADD | |
| BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADDXri)) | |
| .add(MI.getOperand(0)) | |
| .add(MI.getOperand(1)) | |
| .add(MI.getOperand(2)) | |
| .addImm(0); | |
| MI.eraseFromParent(); | |
| return true; | |
| case AArch64::MOVbaseTLS: { | |
| Register DstReg = MI.getOperand(0).getReg(); | |
| auto SysReg = AArch64SysReg::TPIDR_EL0; | |
| MachineFunction *MF = MBB.getParent(); | |
| if (MF->getSubtarget<AArch64Subtarget>().useEL3ForTP()) | |
| SysReg = AArch64SysReg::TPIDR_EL3; | |
| else if (MF->getSubtarget<AArch64Subtarget>().useEL2ForTP()) | |
| SysReg = AArch64SysReg::TPIDR_EL2; | |
| else if (MF->getSubtarget<AArch64Subtarget>().useEL1ForTP()) | |
| SysReg = AArch64SysReg::TPIDR_EL1; | |
| BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::MRS), DstReg) | |
| .addImm(SysReg); | |
| MI.eraseFromParent(); | |
| return true; | |
| } | |
| case AArch64::MOVi32imm: | |
| return expandMOVImm(MBB, MBBI, 32); | |
| case AArch64::MOVi64imm: | |
| return expandMOVImm(MBB, MBBI, 64); | |
| case AArch64::RET_ReallyLR: { | |
| // Hiding the LR use with RET_ReallyLR may lead to extra kills in the | |
| // function and missing live-ins. We are fine in practice because callee | |
| // saved register handling ensures the register value is restored before | |
| // RET, but we need the undef flag here to appease the MachineVerifier | |
| // liveness checks. | |
| MachineInstrBuilder MIB = | |
| BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::RET)) | |
| .addReg(AArch64::LR, RegState::Undef); | |
| transferImpOps(MI, MIB, MIB); | |
| MI.eraseFromParent(); | |
| return true; | |
| } | |
| case AArch64::CMP_SWAP_8: | |
| return expandCMP_SWAP(MBB, MBBI, AArch64::LDAXRB, AArch64::STLXRB, | |
| AArch64::SUBSWrx, | |
| AArch64_AM::getArithExtendImm(AArch64_AM::UXTB, 0), | |
| AArch64::WZR, NextMBBI); | |
| case AArch64::CMP_SWAP_16: | |
| return expandCMP_SWAP(MBB, MBBI, AArch64::LDAXRH, AArch64::STLXRH, | |
| AArch64::SUBSWrx, | |
| AArch64_AM::getArithExtendImm(AArch64_AM::UXTH, 0), | |
| AArch64::WZR, NextMBBI); | |
| case AArch64::CMP_SWAP_32: | |
| return expandCMP_SWAP(MBB, MBBI, AArch64::LDAXRW, AArch64::STLXRW, | |
| AArch64::SUBSWrs, | |
| AArch64_AM::getShifterImm(AArch64_AM::LSL, 0), | |
| AArch64::WZR, NextMBBI); | |
| case AArch64::CMP_SWAP_64: | |
| return expandCMP_SWAP(MBB, MBBI, | |
| AArch64::LDAXRX, AArch64::STLXRX, AArch64::SUBSXrs, | |
| AArch64_AM::getShifterImm(AArch64_AM::LSL, 0), | |
| AArch64::XZR, NextMBBI); | |
| case AArch64::CMP_SWAP_128: | |
| case AArch64::CMP_SWAP_128_RELEASE: | |
| case AArch64::CMP_SWAP_128_ACQUIRE: | |
| case AArch64::CMP_SWAP_128_MONOTONIC: | |
| return expandCMP_SWAP_128(MBB, MBBI, NextMBBI); | |
| case AArch64::AESMCrrTied: | |
| case AArch64::AESIMCrrTied: { | |
| MachineInstrBuilder MIB = | |
| BuildMI(MBB, MBBI, MI.getDebugLoc(), | |
| TII->get(Opcode == AArch64::AESMCrrTied ? AArch64::AESMCrr : | |
| AArch64::AESIMCrr)) | |
| .add(MI.getOperand(0)) | |
| .add(MI.getOperand(1)); | |
| transferImpOps(MI, MIB, MIB); | |
| MI.eraseFromParent(); | |
| return true; | |
| } | |
| case AArch64::IRGstack: { | |
| MachineFunction &MF = *MBB.getParent(); | |
| const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); | |
| const AArch64FrameLowering *TFI = | |
| MF.getSubtarget<AArch64Subtarget>().getFrameLowering(); | |
| // IRG does not allow immediate offset. getTaggedBasePointerOffset should | |
| // almost always point to SP-after-prologue; if not, emit a longer | |
| // instruction sequence. | |
| int BaseOffset = -AFI->getTaggedBasePointerOffset(); | |
| Register FrameReg; | |
| StackOffset FrameRegOffset = TFI->resolveFrameOffsetReference( | |
| MF, BaseOffset, false /*isFixed*/, false /*isSVE*/, FrameReg, | |
| /*PreferFP=*/false, | |
| /*ForSimm=*/true); | |
| Register SrcReg = FrameReg; | |
| if (FrameRegOffset) { | |
| // Use output register as temporary. | |
| SrcReg = MI.getOperand(0).getReg(); | |
| emitFrameOffset(MBB, &MI, MI.getDebugLoc(), SrcReg, FrameReg, | |
| FrameRegOffset, TII); | |
| } | |
| BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::IRG)) | |
| .add(MI.getOperand(0)) | |
| .addUse(SrcReg) | |
| .add(MI.getOperand(2)); | |
| MI.eraseFromParent(); | |
| return true; | |
| } | |
| case AArch64::TAGPstack: { | |
| int64_t Offset = MI.getOperand(2).getImm(); | |
| BuildMI(MBB, MBBI, MI.getDebugLoc(), | |
| TII->get(Offset >= 0 ? AArch64::ADDG : AArch64::SUBG)) | |
| .add(MI.getOperand(0)) | |
| .add(MI.getOperand(1)) | |
| .addImm(std::abs(Offset)) | |
| .add(MI.getOperand(4)); | |
| MI.eraseFromParent(); | |
| return true; | |
| } | |
| case AArch64::STGloop_wback: | |
| case AArch64::STZGloop_wback: | |
| return expandSetTagLoop(MBB, MBBI, NextMBBI); | |
| case AArch64::STGloop: | |
| case AArch64::STZGloop: | |
| report_fatal_error( | |
| "Non-writeback variants of STGloop / STZGloop should not " | |
| "survive past PrologEpilogInserter."); | |
| case AArch64::STR_ZZZZXI: | |
| return expandSVESpillFill(MBB, MBBI, AArch64::STR_ZXI, 4); | |
| case AArch64::STR_ZZZXI: | |
| return expandSVESpillFill(MBB, MBBI, AArch64::STR_ZXI, 3); | |
| case AArch64::STR_ZZXI: | |
| return expandSVESpillFill(MBB, MBBI, AArch64::STR_ZXI, 2); | |
| case AArch64::LDR_ZZZZXI: | |
| return expandSVESpillFill(MBB, MBBI, AArch64::LDR_ZXI, 4); | |
| case AArch64::LDR_ZZZXI: | |
| return expandSVESpillFill(MBB, MBBI, AArch64::LDR_ZXI, 3); | |
| case AArch64::LDR_ZZXI: | |
| return expandSVESpillFill(MBB, MBBI, AArch64::LDR_ZXI, 2); | |
| case AArch64::BLR_RVMARKER: | |
| return expandCALL_RVMARKER(MBB, MBBI); | |
| case AArch64::BLR_BTI: | |
| return expandCALL_BTI(MBB, MBBI); | |
| case AArch64::StoreSwiftAsyncContext: | |
| return expandStoreSwiftAsyncContext(MBB, MBBI); | |
| case AArch64::RestoreZAPseudo: { | |
| auto *NewMBB = expandRestoreZA(MBB, MBBI); | |
| if (NewMBB != &MBB) | |
| NextMBBI = MBB.end(); // The NextMBBI iterator is invalidated. | |
| return true; | |
| } | |
| case AArch64::MSRpstatePseudo: { | |
| auto *NewMBB = expandCondSMToggle(MBB, MBBI); | |
| if (NewMBB != &MBB) | |
| NextMBBI = MBB.end(); // The NextMBBI iterator is invalidated. | |
| return true; | |
| } | |
| case AArch64::OBSCURE_COPY: { | |
| if (MI.getOperand(0).getReg() != MI.getOperand(1).getReg()) { | |
| BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ORRXrs)) | |
| .add(MI.getOperand(0)) | |
| .addReg(AArch64::XZR) | |
| .add(MI.getOperand(1)) | |
| .addImm(0); | |
| } | |
| MI.eraseFromParent(); | |
| return true; | |
| } | |
| } | |
| return false; | |
| } | |
| /// Iterate over the instructions in basic block MBB and expand any | |
| /// pseudo instructions. Return true if anything was modified. | |
| bool AArch64ExpandPseudo::expandMBB(MachineBasicBlock &MBB) { | |
| bool Modified = false; | |
| MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end(); | |
| while (MBBI != E) { | |
| MachineBasicBlock::iterator NMBBI = std::next(MBBI); | |
| Modified |= expandMI(MBB, MBBI, NMBBI); | |
| MBBI = NMBBI; | |
| } | |
| return Modified; | |
| } | |
| bool AArch64ExpandPseudo::runOnMachineFunction(MachineFunction &MF) { | |
| TII = static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo()); | |
| bool Modified = false; | |
| for (auto &MBB : MF) | |
| Modified |= expandMBB(MBB); | |
| return Modified; | |
| } | |
| /// Returns an instance of the pseudo instruction expansion pass. | |
| FunctionPass *llvm::createAArch64ExpandPseudoPass() { | |
| return new AArch64ExpandPseudo(); | |
| } |